/*
 * harness.c -- KLD harness for the pass-2 audit of sys/netbt/hci_unit.c
 * (findings DF-5752 / DF-5753 / DF-5754).
 *
 * The production files hci_unit.c / bt_input.c / hci_event.c / hci_link.c /
 * hci_misc.c / l2cap_lower.c / sco_upper.c are compiled into this module
 * VERBATIM (copied from /usr/src/sys/netbt by build.sh, md5s recorded).
 * This harness plays the role the controller driver (ubt/btuart) plays in
 * production: it attaches a unit via the production hci_attach() and
 * injects synthetic HCI packets through the production entry points
 *
 *	hci_input_event() / hci_complete_sco()  ->  netisr(NETISR_BLUETOOTH)
 *	  -> btintr() -> hci_intr() -> hci_event()/hci_sco_complete()
 *
 * Triggers (sysctl, as root):
 *
 *   hw.hciu_probe = 1
 *	 print struct offsets / cache facts used by the analysis
 *
 *   hw.hciu_setup = 1
 *	 attach fake unit + ACL link #5 (OPEN) + rig l2cap_channel
 *
 *   [DF-5752] netbt hands each rx mbuf to netisr_queue() (embedding a
 *   netmsg inside the mbuf at m_hdr.mh_netmsg, offset 40) AND keeps the
 *   same mbuf on the unit's private ifqueue; hci_intr() drains the
 *   private queue and hci_event() FREES the mbufs while 1..N-1 of their
 *   embedded netmsgs are still linked on the netisr thread's port queue.
 *   The port queue then operates through freed memory:
 *
 *   hw.hciu_baseline = n
 *	 inject n filler events back-to-back, passive only.  The harness
 *	 hci_mtap() hook (called by hci_intr() for every packet, before
 *	 hci_event() frees the previous ones) prints
 *	 "PROOF: mbuf freed, netmsg still port-queued" lines: the freed
 *	 mbuf's lwkt_msg still has MSGF_QUEUED set => the netisr port
 *	 will _lwkt_pullmsg()/dispatch() through freed memory.
 *
 *   hw.hciu_fire = n
 *	 same burst, but at the 3rd processed packet hci_mtap() also
 *	 re-injects 3 more events FROM NETISR CONTEXT (a controller rx
 *	 interrupt on the same CPU in production).  The m_gethdr()s pop
 *	 the just-freed chunks (objcache LIFO) whose netmsgs are STILL
 *	 QUEUED => netisr_queue() re-initialises a live/dangling netmsg
 *	 and TAILQ_INSERT_TAILs the SAME node a second time => the port
 *	 queue linkage is corrupted; the queue chain is dumped after.
 *
 *   [DF-5753] hw.hcius_fire = rounds
 *	 per round: production sco_attach() + SCO link; fire 200 SCO
 *	 packets of exactly sp_mtu bytes via production sco_send() (the
 *	 fake controller completes them instantly via production
 *	 hci_complete_sco() -> hci_scodone + netisr, exactly what a USB
 *	 completion interrupt does); then production sco_disconnect
 *	 (pcb, 0) -- what sco_sabort() calls on socket close -- which
 *	 NULLs link->hl_sco and runs hci_link_free() on the syscall
 *	 thread.  The netisr thread's hci_intr() scodone walk matches
 *	 the still-listed link and calls hci_sco_complete() ->
 *	 link->hl_sco->sp_pending-- with hl_sco == NULL -> NULL write.
 *
 *   [DF-5754] hw.hciud_fire = rounds
 *	 per round: hci_attach() a fresh unit, flood hci_input_event()
 *	 from a kthread (16-response inquiry events -> slow drain), and
 *	 call production hci_detach() concurrently -> hci_disable() +
 *	 TAILQ_REMOVE + kfree(unit) while btintr()/hci_intr() walk the
 *	 unit list / process the queues -> use-after-free.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/types.h>
#include <sys/endian.h>
#include <sys/msgport.h>
#include <sys/msgport2.h>

#include <net/netisr.h>

#include <netbt/bluetooth.h>
#include <netbt/hci.h>
#include <netbt/l2cap.h>
#include <netbt/sco.h>

struct hci_unit *g_unit;

static volatile int g_d_rx_on;	/* fake controller rx enabled (DF-5754) */
static volatile int g_d_inflight;

/* ------------------------------------------------------------------ */
/* fake controller driver: the role ubt(4)/btuart(4) play             */

static int	 fake_enable(device_t);
static void	 fake_disable(device_t);
static void	 fake_output_cmd(device_t, struct mbuf *);
static void	 fake_output_acl(device_t, struct mbuf *);
static void	 fake_output_sco(device_t, struct mbuf *);
static void	 fake_get_stats(device_t, struct bt_stats *, int);

static const struct hci_if fake_hci_if = {
	fake_enable,
	fake_disable,
	fake_output_cmd,
	fake_output_acl,
	fake_output_sco,
	fake_get_stats,
};

static int
fake_enable(device_t dev)
{
	return 0;
}

static void
fake_disable(device_t dev)
{
	/*
	 * A real controller driver stops its rx paths here and waits for
	 * in-flight rx callbacks to return before disable returns
	 * (usb transfer abort + drain).  The DF-5754 rig honours that
	 * contract; what remains racing hci_detach() afterwards is the
	 * KERNEL side: btintr()/hci_intr() processing queued netmsgs on
	 * the netisr thread -- exactly what the DF-5754 fix protects.
	 */
	g_d_rx_on = 0;
	while (g_d_inflight > 0)
		tsleep(&g_d_inflight, 0, "hciqsc", 1);
}

static void
fake_output_cmd(device_t dev, struct mbuf *m)
{
	m_freem(m);
}

static void
fake_output_acl(device_t dev, struct mbuf *m)
{
	m_freem(m);
}

static void
fake_output_sco(device_t dev, struct mbuf *m)
{
	/* the controller "sends" the packet and the USB completion
	 * interrupt fires immediately: hand the mbuf to the production
	 * completion entry point (ubt's callback -> hci_complete_sco) */
	hci_complete_sco(g_unit, m);	/* production */
}

static void
fake_get_stats(device_t dev, struct bt_stats *s, int arg)
{
	memset(s, 0, sizeof(*s));
}

/* ------------------------------------------------------------------ */

static struct hci_link *g_link;

#define TEST_ACL_HANDLE	0x0005
#define TEST_SCO_HANDLE	0x0009

/* rig L2CAP channel (not needed for the DF-5752 proof, kept for
 * completeness of the unit state) */
static struct l2cap_channel g_chan;

/* rig SCO upper layer */
static void rig_sco_complete(void *, int);
static void rig_sco_disconnected(void *, int);
static void rig_sco_input(void *, struct mbuf *);

static const struct btproto rig_sco_proto = {
	NULL,			/* connecting */
	NULL,			/* connected */
	rig_sco_disconnected,	/* disconnected */
	NULL,			/* newconn */
	rig_sco_complete,	/* complete */
	NULL,			/* linkmode */
	rig_sco_input,		/* input */
};

static int g_sco_completes, g_sco_disconnects;

static void
rig_sco_complete(void *arg, int num)
{
	g_sco_completes++;
}

static void
rig_sco_disconnected(void *arg, int err)
{
	g_sco_disconnects++;
}

static void
rig_sco_input(void *arg, struct mbuf *m)
{
	m_freem(m);
}

/* ------------------------------------------------------------------ */
/* DF-5752 tracking                                                    */

struct rig_ent {
	struct mbuf	*m;
	lwkt_msg_t	 msg;
	int		 seen;		/* hci_mtap called for it */
};

static struct rig_ent rig_tab[64];
static int rig_n;
static int rig_mtap_count;
static int rig_reinject_at = 3;	/* hci_mtap call # that re-injects */
static int rig_reinject_on;	/* only enabled by hciuu_fire=1 */
static int rig_dangling_proof;
static int rig_reused_hit;

/* in-memory capture of dangling observations (console kprintf from the
 * netisr thread can be dropped by the serial driver, so evidence is
 * recorded here and printed from the syscall thread afterwards) */
struct rig_dangling {
	struct mbuf	*m;
	lwkt_msg_t	 msg;
	uint32_t	 flags;
	int		 mtap_call;
};
static struct rig_dangling rig_dang[16];
static int rig_dang_n;

static void
rig_track(struct mbuf *m)
{
	if (rig_n < (int)NELEM(rig_tab)) {
		rig_tab[rig_n].m = m;
		rig_tab[rig_n].msg = &m->m_hdr.mh_netmsg.base.lmsg;
		rig_tab[rig_n].seen = 0;
		rig_n++;
	}
}

/*
 * harness stand-in for hci_socket.c's hci_mtap(): hci_intr() calls this
 * for EVERY packet it dequeues, before hci_event()/hci_sco_recv()/
 * hci_acl_recv() processes (and frees) it.  Runs in netisr context.
 */
void
hci_mtap(struct mbuf *m, struct hci_unit *unit)
{
	int i;

	rig_mtap_count++;

	/* all previously-seen packets have been processed and freed by
	 * hci_event() (m_freem at hci_event.c:229) when we get here */
	for (i = 0; i < rig_n; i++) {
		if (rig_tab[i].seen && rig_tab[i].m != m &&
		    (rig_tab[i].msg->ms_flags & MSGF_QUEUED)) {
			/* freed mbuf, netmsg still linked on the port */
			if (rig_dang_n < (int)NELEM(rig_dang)) {
				rig_dang[rig_dang_n].m = rig_tab[i].m;
				rig_dang[rig_dang_n].msg = rig_tab[i].msg;
				rig_dang[rig_dang_n].flags =
				    rig_tab[i].msg->ms_flags;
				rig_dang[rig_dang_n].mtap_call =
				    rig_mtap_count;
				rig_dang_n++;
			}
			rig_dangling_proof++;
		}
	}

	/* mark this packet seen (it will be freed right after we return) */
	for (i = 0; i < rig_n; i++) {
		if (rig_tab[i].m == m) {
			rig_tab[i].seen = 1;
			break;
		}
	}

	/*
	 * mid-drain re-injection (a controller rx interrupt landing on the
	 * same CPU in production): the allocations here pop the chunks the
	 * drain just freed (objcache LIFO).  If a popped chunk's netmsg is
	 * still port-queued, netisr_queue() re-initialises it and inserts
	 * the same node again => double-linked queue corruption.
	 */
	if (rig_reinject_on && rig_mtap_count == rig_reinject_at) {
		static const uint8_t fill[] = { 0x04, 0x02, 0x01, 0x00 };
		struct mbuf *nm;
		int k;

		kprintf("[harness] hci_mtap: mid-drain re-injection of 3 "
		    "events from netisr context (objcache LIFO reuse of "
		    "just-freed chunks)\n");
		for (k = 0; k < 3; k++) {
			nm = m_gethdr(M_NOWAIT, MT_DATA);
			KKASSERT(nm != NULL);
			if (m_copyback2(nm, 0, sizeof(fill), fill,
			    M_NOWAIT) != 0)
				panic("copyback");
			/* does this chunk alias a tracked netmsg? */
			{
				int j;

				for (j = 0; j < rig_n; j++) {
					if (rig_tab[j].seen &&
					    (void *)nm == (void *)rig_tab[j].m) {
						kprintf("[harness] REUSE HIT: "
						    "new mbuf %p == freed "
						    "tracked mbuf idx %d "
						    "(netmsg %p flags=0x%x "
						    "QUEUED=%d)\n", nm, j,
						    rig_tab[j].msg,
						    rig_tab[j].msg->ms_flags,
						    !!(rig_tab[j].msg->
						    ms_flags & MSGF_QUEUED));
						rig_reused_hit++;
					}
				}
			}
			hci_input_event(unit, nm);	/* production */
		}
	}
}

void
hci_drop(void *arg)
{
	/* no socket bookkeeping to drop */
}

/* ------------------------------------------------------------------ */
/* probe                                                               */

static int
hciuu_probe(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err;
	struct mbuf *m;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	kprintf("[probe] sizeof(struct mbuf)     = %zu\n", sizeof(struct mbuf));
	kprintf("[probe] sizeof(struct pkthdr)   = %zu\n", sizeof(struct pkthdr));
	kprintf("[probe] sizeof(struct m_hdr)    = %zu\n", sizeof(struct m_hdr));
	kprintf("[probe] sizeof(lwkt_msg)        = %zu\n", sizeof(lwkt_msg));
	kprintf("[probe] sizeof(struct netmsg_packet) = %zu\n",
	    sizeof(struct netmsg_packet));
	m = m_gethdr(M_WAITOK, MT_DATA);
	KKASSERT(m != NULL);
	kprintf("[probe] m=%p mh_netmsg@%td lmsg@%td nm_dispatch@%td "
	    "nm_packet@%td\n", m,
	    (char *)&m->m_hdr.mh_netmsg - (char *)m,
	    (char *)&m->m_hdr.mh_netmsg.base.lmsg - (char *)m,
	    (char *)&m->m_hdr.mh_netmsg.base.nm_dispatch - (char *)m,
	    (char *)&m->m_hdr.mh_netmsg.nm_packet - (char *)m);
	kprintf("[probe]   ms_flags@%td (m_pkthdr.rcvif@%td -- separate "
	    "union: no overlay)\n",
	    (char *)&m->m_hdr.mh_netmsg.base.lmsg.ms_flags - (char *)m,
	    (char *)&m->m_pkthdr.rcvif - (char *)m);
	kprintf("[probe] m_pktdat - (char *)m    = %td (MHLEN=%d)\n",
	    (long)(m->m_pktdat - (char *)m), (int)MHLEN);
	m_freem(m);
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, hciu_probe, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, hciuu_probe, "I", "print struct offsets");

/* ------------------------------------------------------------------ */
/* setup                                                               */

static int
hciuu_setup(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	if (g_unit != NULL) {
		kprintf("[harness] unit already attached\n");
		return 0;
	}

	kprintf("[harness] === setup: attach unit + ACL link #%d ===\n",
	    TEST_ACL_HANDLE);

	g_unit = hci_attach(&fake_hci_if, (device_t)(uintptr_t)0xcafe0000UL, 0);
	KKASSERT(g_unit != NULL);

	g_unit->hci_flags |= BTF_RUNNING;
	g_unit->hci_num_cmd_pkts = 1;
	g_unit->hci_max_acl_size = 32;
	g_unit->hci_num_acl_pkts = 0;

	g_link = hci_link_alloc(g_unit);
	KKASSERT(g_link != NULL);
	g_link->hl_type = HCI_LINK_ACL;
	g_link->hl_state = HCI_LINK_OPEN;
	g_link->hl_handle = TEST_ACL_HANDLE;

	memset(&g_chan, 0, sizeof(g_chan));
	g_chan.lc_lcid = 0x0041;
	g_chan.lc_rcid = 0x0042;
	g_chan.lc_state = L2CAP_OPEN;
	g_chan.lc_link = g_link;
	g_chan.lc_txq.ifq_maxlen = 50;

	kprintf("[harness] unit %p link %p attached\n", g_unit, g_link);
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, hciu_setup, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, hciuu_setup, "I", "attach fake HCI unit + ACL link");

/* ------------------------------------------------------------------ */
/* DF-5752                                                             */

static void
inject_event(const uint8_t *pkt, int len)
{
	struct mbuf *m;

	m = m_gethdr(M_NOWAIT, MT_DATA);
	KKASSERT(m != NULL);
	if (m_copyback2(m, 0, len, pkt, M_NOWAIT) != 0)
		panic("copyback");

	rig_track(m);
	hci_input_event(g_unit, m);	/* production entry point */
}

static void
rig_reset(void)
{
	rig_n = 0;
	rig_mtap_count = 0;
	rig_dangling_proof = 0;
	rig_reused_hit = 0;
	rig_dang_n = 0;
}

static int
hciuu_fire(SYSCTL_HANDLER_ARGS)
{
	/* filler: Inquiry Result event with zero responses (cheap, no-op) */
	static const uint8_t fill[] = { 0x04, 0x02, 0x01, 0x00 };
	int v = 0, err, i;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	KKASSERT(g_unit != NULL);

	if (v == 2) {
		/* baseline: passive, no re-injection */
		rig_reinject_on = 0;
		kprintf("[harness] === DF-5752 baseline: %d filler events "
		    "back-to-back (passive dangling proof) ===\n", 8);
	} else {
		rig_reinject_on = 1;
		kprintf("[harness] === DF-5752 fire: 8 fillers + mid-drain "
		    "re-injection at packet #%d ===\n", rig_reinject_at);
	}

	rig_reset();
	for (i = 0; i < 8; i++)
		inject_event(fill, sizeof(fill));

	kprintf("[harness] burst injected; waiting 3s for netisr...\n");
	tsleep(&rig_n, 0, "hciuu", 3 * hz);
	kprintf("[harness] done: dangling_observations=%d reuse_hits=%d "
	    "mtap_calls=%d\n", rig_dangling_proof, rig_reused_hit,
	    rig_mtap_count);
	for (i = 0; i < rig_dang_n; i++)
		kprintf("[harness] DF-5752 PROOF: at hci_mtap call #%d: "
		    "mbuf %p was already freed by hci_event() "
		    "(hci_event.c m_freem) but its netmsg %p still had "
		    "MSGF_QUEUED set -- the netisr port queue is holding "
		    "and will _lwkt_pullmsg()/TAILQ_REMOVE/dispatch() "
		    "through freed memory (flags=0x%x)\n",
		    rig_dang[i].mtap_call, rig_dang[i].m, rig_dang[i].msg,
		    rig_dang[i].flags);
	if (rig_reused_hit)
		kprintf("[harness] REUSE: freed chunks whose netmsgs were "
		    "still port-queued were reallocated and re-inserted "
		    "(same node linked twice: port queue corrupted)\n");
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, hciu_fire, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, hciuu_fire, "I", "fire DF-5752 burst (2=baseline)");

static int
hciuu_dump(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err, i;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	for (i = 0; i < rig_n; i++)
		kprintf("[dump] idx=%d m=%p msg=%p seen=%d flags=0x%x "
		    "QUEUED=%d\n", i, rig_tab[i].m, rig_tab[i].msg,
		    rig_tab[i].seen, rig_tab[i].msg->ms_flags,
		    !!(rig_tab[i].msg->ms_flags & MSGF_QUEUED));
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, hciu_dump, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, hciuu_dump, "I", "dump DF-5752 tracking table");

/* ------------------------------------------------------------------ */
/* DF-5753: scodone dispatch vs syscall-context link teardown          */

static int
hcius_fire(SYSCTL_HANDLER_ARGS)
{
	struct sco_pcb *pcb = NULL;
	struct hci_link *sco;
	struct mbuf *m;
	uint8_t buf[32];
	int v = 0, err, round, sent;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	KKASSERT(g_unit != NULL);

	kprintf("[harness] === DF-5753: %d rounds of [200 SCO sends + "
	    "production sco_disconnect(pcb,0)] ===\n", v);
	kprintf("[harness] each send: sco_send() M_SETCTX(m, link); fake "
	    "controller completes instantly via hci_complete_sco(); "
	    "sco_disconnect NULLs hl_sco then runs hci_link_free() on this "
	    "(syscall) thread while netisr hci_intr() walks hci_links "
	    "matching M_GETCTX -> hci_sco_complete() derefs hl_sco==NULL\n");

	for (round = 0; round < v; round++) {
		/* fresh pcb + SCO link each round */
		err = sco_attach(&pcb, &rig_sco_proto, NULL);	/* production */
		KKASSERT(err == 0 && pcb != NULL);

		sco = hci_link_alloc(g_unit);			/* production */
		KKASSERT(sco != NULL);
		sco->hl_type = HCI_LINK_SCO;
		sco->hl_state = HCI_LINK_OPEN;
		sco->hl_handle = TEST_SCO_HANDLE;
		sco->hl_sco = pcb;
		pcb->sp_link = sco;
		pcb->sp_mtu = sizeof(buf);

		/* pile up completions (each queues scodone + netisr kick) */
		for (sent = 0; sent < 200; sent++) {
			m = m_gethdr(M_NOWAIT, MT_DATA);
			KKASSERT(m != NULL);
			if (m_copyback2(m, 0, sizeof(buf), buf, M_NOWAIT) != 0)
				panic("copyback");
			err = sco_send(pcb, m);		/* production */
			if (err)
				panic("sco_send %d", err);
		}

		/* the socket-close path: sco_sabort() -> sco_disconnect(,0) */
		sco_disconnect(pcb, 0);			/* production */

		if (round % 25 == 0)
			kprintf("[harness] round %d ok (completes=%d "
			    "disconnects=%d)\n", round, g_sco_completes,
			    g_sco_disconnects);
	}

	kprintf("[harness] all rounds done without a hit (race not won; "
	    "rerun with more rounds)\n");
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, hcius_fire, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, hcius_fire, "I", "fire DF-5753 scodone teardown race");

/* ------------------------------------------------------------------ */
/* DF-5754: hci_detach unit lifetime vs btintr/hci_intr                */

static struct hci_unit *g_d_unit;
static int g_d_run;

static int hciud_mode = 1;	/* 1 = filler events (no memo allocs:
				 * isolates the unit-lifetime race from the
				 * known DF-0683 disable-drain race),
				 * 2 = inquiry-16 (baseline payload) */

static void
ud_injector_thread(void *arg)
{
	/* 16-response Inquiry Result: slow hci_event processing widens the
	 * hci_intr drain window */
	uint8_t pkt[3 + 1 + 16 * 14];
	uint8_t fill[] = { 0x04, 0x02, 0x01, 0x00 };	/* Inquiry, 0 resp */
	const uint8_t *send;
	int sendlen;
	int off = 0, resp, i;
	static uint16_t bdctr;

	if (hciud_mode == 2) {
		pkt[off++] = 0x04;
		pkt[off++] = 0x02;			/* INQUIRY_RESULT */
		pkt[off++] = 1 + 16 * 14;
		pkt[off++] = 16;
		for (resp = 0; resp < 16; resp++) {
			pkt[off++] = bdctr & 0xff;
			pkt[off++] = (bdctr >> 8) & 0xff;
			bdctr++;
			for (i = 0; i < 4; i++)
				pkt[off++] = 0xc0 + i;
			pkt[off++] = 0x01;
			pkt[off++] = 0x00;
			pkt[off++] = 0x00;
			pkt[off++] = 0x22;
			pkt[off++] = 0x22;
			pkt[off++] = 0x22;
			pkt[off++] = 0x11;
			pkt[off++] = 0x11;
		}
		KKASSERT(off == (int)sizeof(pkt));
		send = pkt;
		sendlen = sizeof(pkt);
	} else {
		send = fill;
		sendlen = sizeof(fill);
	}

	while (1) {
		struct mbuf *m;

		/* increment BEFORE checking the enable flag so that
		 * fake_disable()'s inflight drain cannot pass between our
		 * check and our call (real drivers serialize this the same
		 * way around their transfer-arming path) */
		atomic_add_int(&g_d_inflight, 1);
		if (!g_d_run || !g_d_rx_on) {
			atomic_add_int(&g_d_inflight, -1);
			break;
		}

		m = m_gethdr(M_NOWAIT, MT_DATA);
		if (m == NULL) {
			atomic_add_int(&g_d_inflight, -1);
			continue;
		}
		if (m_copyback2(m, 0, sendlen, send, M_NOWAIT) != 0) {
			m_freem(m);
			atomic_add_int(&g_d_inflight, -1);
			continue;
		}
		hci_input_event(g_d_unit, m);	/* production */
		atomic_add_int(&g_d_inflight, -1);
	}
	kprintf("[harness] injector stopped\n");
	lwkt_exit();
}

static int
hciud_fire(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err, round;
	struct thread *td = NULL;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	hciud_mode = (v == 2) ? 2 : 1;
	v = (v == 2) ? 200 : v;	/* inquiry mode: 200 slow rounds */

	KKASSERT(g_unit != NULL);	/* ensures netisr registered */

	kprintf("[harness] === DF-5754: %d rounds of [attach + flood-inject "
	    "+ hci_detach] ===\n", v);
	kprintf("[harness] hci_detach() runs hci_disable + TAILQ_REMOVE + "
	    "kfree on this thread while btintr() walks hci_unit_list and "
	    "hci_intr() drains the unit's queues on the netisr thread\n");

	for (round = 0; round < v; round++) {
		g_d_unit = hci_attach(&fake_hci_if,
		    (device_t)(uintptr_t)(0xcafe1000UL + round), 0);
		KKASSERT(g_d_unit != NULL);
		g_d_unit->hci_flags |= BTF_RUNNING;
		g_d_unit->hci_num_cmd_pkts = 1;
		g_d_rx_on = 1;

		g_d_run = 1;
		err = lwkt_create(ud_injector_thread, NULL, &td, NULL,
		    TDF_NOSTART, -1, "hciud_inj");
		KKASSERT(err == 0);
		lwkt_schedule(td);

		/* let the flood start, then tear the unit down under it
		 * (fake_disable quiesces rx per the driver contract; the
		 * netisr-side btintr/hci_intr race remains fully live) */
		tsleep(&g_d_run, 0, "hciud", hz / 100 + 1);

		hci_detach(g_d_unit);		/* production */
		g_d_run = 0;
		g_d_unit = NULL;

		if (round % 20 == 0)
			kprintf("[harness] round %d survived\n", round);
		tsleep(&g_d_run, 0, "hciud", hz / 50 + 1);
	}

	kprintf("[harness] all rounds done without a hit\n");
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, hciud_fire, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, hciud_fire, "I", "fire DF-5754 detach lifetime race");

/* ------------------------------------------------------------------ */

static int
harness_mode(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		l2cap_pdu_pool = zinit("l2cap_pdu",
		    sizeof(struct l2cap_pdu), 1, ZONE_DESTROYABLE);
		l2cap_req_pool = zinit("l2cap_req",
		    sizeof(struct l2cap_req), 1, ZONE_DESTROYABLE);
		KKASSERT(l2cap_pdu_pool != NULL && l2cap_req_pool != NULL);

		netisr_register(NETISR_BLUETOOTH, btintr, NULL);

		kprintf("[harness] hci_unit_harness loaded: hciu_probe, "
		    "hciuu_setup, hciu_fire, hciu_dump, hcius_fire, "
		    "hciud_fire\n");
		break;
	case MOD_UNLOAD:
		return EBUSY;
	default:
		break;
	}
	return 0;
}

static moduledata_t harness_moddata = {
	"hci_unit_harness",
	harness_mode,
	NULL
};

DECLARE_MODULE(hci_unit_harness, harness_moddata, SI_SUB_DRIVERS,
    SI_ORDER_MIDDLE);