/*
 * harness.c -- KLD harness for the pass-2 audit of sys/netbt/l2cap_upper.c
 * (finding DF-5749).
 *
 * The production files l2cap_upper.c / l2cap_lower.c / l2cap_misc.c /
 * l2cap_signal.c are compiled into this module VERBATIM (copied from
 * /usr/src/sys/netbt by build.sh, md5s recorded).  This harness plays the
 * two roles that reach l2cap_detach() in production:
 *
 *   close(2):    syscall thread -> netmsg -> cpu0 netisr port ->
 *                l2cap_sdetach() -> l2cap_detach()  [sys/netbt/l2cap_socket.c]
 *   rtx timeout: softclock thread -> l2cap_rtx()    [sys/netbt/l2cap_misc.c]
 *
 * There is no locking between those two contexts anywhere in netbt.
 * When the response-timeout callout fires while the socket is being
 * detached, both contexts execute l2cap_close() on the same channel:
 *
 *   l2cap_detach()  -> l2cap_close() walks chan->lc_link->hl_reqs and
 *                      calls l2cap_request_free() -> callout_stop() which
 *                      SYNCHRONOUSLY WAITS for the in-flight l2cap_rtx()
 *                      (kern_timeout.c).  l2cap_rtx() proceeds to run
 *                      l2cap_close(chan, ETIMEDOUT) itself -- state CLOSED,
 *                      hci_acl_close(link) frees the ACL link,
 *                      chan->lc_link = NULL -- and returns.  The detach
 *                      context now resumes INSIDE l2cap_close() after the
 *                      request walk and executes the body a SECOND time:
 *                      hci_acl_close(chan->lc_link == NULL, err) ->
 *                      "--NULL->hl_refcnt" -> kernel page fault panic
 *                      (KKASSERT(link != NULL) is a no-op on production
 *                      GENERIC).  Co-fires DF-0745 (double
 *                      TAILQ_REMOVE/zfree of the l2cap_req).
 *
 * Channel state built per round is exactly what production
 * l2cap_connect() produces (l2cap_upper.c:139-195): CID allocated, ACL
 * link attached with one reference (hci_acl_open),
 * L2CAP_WAIT_RECV_CONNECT_RSP, one outstanding L2CAP_CONNECT_REQ
 * request with the rtx callout armed -- re-armed to 1 tick, the
 * production hci_acl_close() re-arm pattern (hci_link.c:161-162), just
 * sooner.
 *
 * Triggers (sysctl, as root):
 *
 *   hw.l2uh_setup = 1   print rig/kernel facts (sizes, offsets, hz)
 *
 *   hw.l2uh_once  = 1   CONTROL: one unraced round; rtx fires cleanly,
 *                        then l2cap_detach().  Expect: no panic, exactly
 *                        one disconnected() callback, channel freed.
 *
 *   hw.l2uh_race  = N   RACE: N rounds in bursts of 256.  Each burst:
 *                        arm 256 channels with 1-tick rtx callouts,
 *                        tsleep(1 tick) to land on the tick where all
 *                        256 rtx callbacks fire on the softclock thread,
 *                        then storm l2cap_detach() on all 256 from this
 *                        (syscall) thread.
 *                        EXPECTED: kernel panic -- page fault in
 *                        hci_acl_close/l2cap_close (NULL lc_link), or
 *                        slab/zone corruption from the double free.
 *
 *   hw.l2uh_state = 1   dump counters.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <sys/thread.h>
#include <sys/thread2.h>
#include <sys/endian.h>

#include <netbt/bluetooth.h>
#include <netbt/hci.h>
#include <netbt/l2cap.h>

#include "l2uh.h"

extern const struct btproto stub_btproto;	/* stub.c */

static int	l2uh_setup(SYSCTL_HANDLER_ARGS);
static int	l2uh_once(SYSCTL_HANDLER_ARGS);
static int	l2uh_race(SYSCTL_HANDLER_ARGS);
static int	l2uh_state(SYSCTL_HANDLER_ARGS);

SYSCTL_PROC(_hw, OID_AUTO, l2uh_setup, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_setup, "I", "print rig facts (sizes/offsets)");
SYSCTL_PROC(_hw, OID_AUTO, l2uh_once, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_once, "I", "control: one unraced round");
SYSCTL_PROC(_hw, OID_AUTO, l2uh_race, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_race, "I", "DF-5749: race rtx timeout vs l2cap_detach");
SYSCTL_PROC(_hw, OID_AUTO, l2uh_state, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_state, "I", "dump counters");

static u_long	 n_armed, n_detached;
static int	 burst_panic_note;

/* arm_round phase tracking (stall diagnosis) */
static volatile int arm_phase;		/* 0=idle 1=attach 2=cid 3=link
					 * 4=request_alloc 5=publish */
static volatile int arm_stall_seen;

/*
 * Build the production post-l2cap_connect() channel state.
 * Returns the channel or NULL.
 */
static struct l2cap_channel *
arm_round(void)
{
	struct l2cap_channel *chan;
	struct hci_link *link;
	struct l2cap_req *req;

	arm_phase = 1;
	if (l2cap_attach(&chan, &stub_btproto, NULL) != 0)
		return NULL;

	arm_phase = 2;
	if (l2cap_cid_alloc(chan) != 0) {	/* active list insert */
		kfree(chan, M_BLUETOOTH);
		return NULL;
	}

	arm_phase = 3;
	link = stub_link_new();
	if (link == NULL) {
		chan->lc_lcid = L2CAP_NULL_CID;
		LIST_REMOVE(chan, lc_ncid);
		kfree(chan, M_BLUETOOTH);
		return NULL;
	}
	link->hl_refcnt = 1;			/* the hci_acl_open() ref */
	chan->lc_link = link;

	chan->lc_state = L2CAP_WAIT_RECV_CONNECT_RSP;

	/* production request: allocates the id, TAILQ_INSERTs onto
	 * link->hl_reqs, arms lr_rtx for l2cap_response_timeout * hz */
	arm_phase = 4;
	if (l2cap_request_alloc(chan, L2CAP_CONNECT_REQ) != 0) {
		chan->lc_state = L2CAP_CLOSED;
		hci_acl_close(chan->lc_link, ENOMEM);
		chan->lc_link = NULL;
		chan->lc_lcid = L2CAP_NULL_CID;
		LIST_REMOVE(chan, lc_ncid);
		kfree(chan, M_BLUETOOTH);
		return NULL;
	}

	/* re-arm the SAME production callout to 1 tick: the
	 * hci_acl_close() re-arm pattern, just sooner */
	req = TAILQ_FIRST(&link->hl_reqs);
	KKASSERT(req != NULL && req->lr_chan == chan);
	callout_reset(&req->lr_rtx, 1, l2cap_rtx, req);

	arm_phase = 5;
	n_armed++;
	return chan;
}

static int
l2uh_setup(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	kprintf("[l2uh] sizeof(l2cap_channel)=%zu offsetof(lc_state)=%zu "
	    "offsetof(lc_link)=%zu\n", sizeof(struct l2cap_channel),
	    __offsetof(struct l2cap_channel, lc_state),
	    __offsetof(struct l2cap_channel, lc_link));
	kprintf("[l2uh] sizeof(l2cap_req)=%zu sizeof(hci_link)=%zu "
	    "offsetof(hl_refcnt)=%zu offsetof(hl_txq)=%zu\n",
	    sizeof(struct l2cap_req), sizeof(struct hci_link),
	    __offsetof(struct hci_link, hl_refcnt),
	    __offsetof(struct hci_link, hl_txq));
	kprintf("[l2uh] hz=%d l2cap_response_timeout=%ds hci_acl_expiry=%ds "
	    "ncpu=%d\n", hz, l2cap_response_timeout, hci_acl_expiry,
	    ncpus);
	kprintf("[l2uh] rtx body: l2cap_rtx() -> l2cap_close() runs on the "
	    "softclock thread; l2cap_detach() runs on the netisr port "
	    "(close(2)); no lock serializes them\n");
	return 0;
}

static int
l2uh_once(SYSCTL_HANDLER_ARGS)
{
	struct l2cap_channel *chan;
	int v = 0, err, disc_before;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	kprintf("[l2uh] === control: one unraced round ===\n");
	disc_before = stub_proto_disconnected_count;

	chan = arm_round();
	if (chan == NULL) {
		kprintf("[l2uh] arm failed\n");
		return 0;
	}
	kprintf("[l2uh] armed chan=%p state=%d (WAIT_RECV_CONNECT_RSP=%d) "
	    "-- sleeping 3 ticks for the clean rtx fire\n", chan,
	    chan->lc_state, L2CAP_WAIT_RECV_CONNECT_RSP);
	tsleep(&n_armed, 0, "l2uonce", 3);

	kprintf("[l2uh] after rtx: state=%d (expect CLOSED=%d) "
	    "lc_link=%p (expect NULL) disconnected callbacks=%d (expect +%d)\n",
	    chan->lc_state, L2CAP_CLOSED, chan->lc_link,
	    stub_proto_disconnected_count,
	    stub_proto_disconnected_count - disc_before);

	/* NOTE: the rtx close does NOT free the channel (that is
	 * l2cap_detach's job); it also does NOT remove it from the
	 * active list.  detach now. */
	l2cap_detach(&chan);
	n_detached++;
	kprintf("[l2uh] detached. no panic = control OK\n");
	return 0;
}

#define BURST	256

static int
l2uh_race(SYSCTL_HANDLER_ARGS)
{
	struct l2cap_channel *burst[BURST];
	int v = 0, err, i, n, done = 0;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	kprintf("[l2uh] === DF-5749 race: %d rounds in bursts of %d ===\n",
	    v, BURST);
	kprintf("[l2uh] arm burst -> sleep 1 tick (all rtx fire on softclock)"
	    " -> storm l2cap_detach on the burst from this thread\n");

	while (done < v) {
		n = (v - done < BURST) ? v - done : BURST;

		/* arm the burst */
		for (i = 0; i < n; i++) {
			burst[i] = arm_round();
			if (burst[i] == NULL)
				break;
		}
		n = i;

		/* land on the tick where the whole burst's rtx callouts
		 * fire on the softclock thread */
		tsleep(&n_armed, 0, "l2urace", 1);

		/* the close(2) path, concurrent with the rtx storm */
		for (i = 0; i < n; i++) {
			if (burst[i] != NULL) {
				l2cap_detach(&burst[i]);
				n_detached++;
			}
		}

		done += n;
		if ((done % (BURST * 16)) == 0)
			kprintf("[l2uh] %d rounds: armed=%lu detached=%lu "
			    "disconnected=%d\n", done, n_armed, n_detached,
			    stub_proto_disconnected_count);
	}

	kprintf("[l2uh] RACE RUN COMPLETE - no panic (armed=%lu "
	    "detached=%lu disconnected=%d)\n", n_armed, n_detached,
	    stub_proto_disconnected_count);
	return 0;
}

static int
l2uh_state(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	kprintf("[l2uh] armed=%lu detached=%lu disconnected=%d "
	    "connecting=%d active_list chans=%d arm_phase=%d "
	    "stall_seen=%d\n",
	    n_armed, n_detached, stub_proto_disconnected_count,
	    stub_proto_connecting_count,
	    /* count remaining (leaked) channels on the active list */
	    ({ int _n = 0; struct l2cap_channel *_c;
	       LIST_FOREACH(_c, &l2cap_active_list, lc_ncid) _n++;
	       _n; }), arm_phase, arm_stall_seen);
	if (burst_panic_note)
		kprintf("[l2uh] note: %d\n", burst_panic_note);
	return 0;
}

/* ------------------------------------------------------------------ */
/*
 * DF-5749 race3: cross-CPU simultaneous production calls.
 *
 * The single-threaded burst mode above cannot interleave mid-body: on
 * one CPU the LWKT scheduler serializes the syscall thread against the
 * softclock thread, and the tsleep(1) phase alignment lands the detach
 * storm always fully before or fully after the rtx batch (100k clean
 * rounds demonstrated exactly that).  In PRODUCTION the two contexts
 * are different threads usually on different CPUs (softclock vs the
 * cpu0 netisr port), so their bodies genuinely overlap.
 *
 * race3 reproduces the production topology: a kthread pinned to cpu1
 * spins on a sequence word and fires the PRODUCTION l2cap_rtx() the
 * instant the orchestrator publishes a fresh armed channel; the
 * orchestrator (playing the close(2) -> netisr port role) fires the
 * PRODUCTION l2cap_detach() on it at the same moment.  The 1-tick rtx
 * callout armed by arm_round() stays live as well (the real softclock
 * delivery sometimes joins in).
 */
static volatile u_long	r3_seq;		/* publish barrier */
static volatile u_long	r3_done;
static struct l2cap_channel *volatile r3_chan;
static struct l2cap_req *volatile r3_req;
static volatile int	r3_stop;
static struct thread	*r3_td;

static void
r3_rtx_thread(void *arg)
{
	u_long seen = 0;
	int idle = 0;

	for (;;) {
		while (r3_seq == seen) {
			if (r3_stop) {
				wakeup(&r3_stop);
				lwkt_exit();
				/* NOT REACHED */
			}
			if (++idle > 40000000) {	/* ~ seconds of spin */
				idle = 0;
				tsleep(&r3_seq, 0, "l2r3i", 1);
			}
			cpu_ccfence();
		}
		idle = 0;
		seen = r3_seq;
		if (r3_req != NULL)
			l2cap_rtx((struct l2cap_req *)r3_req);  /* production */
		atomic_add_long(&r3_done, 1);
	}
}

static int
l2uh_race3(SYSCTL_HANDLER_ARGS)
{
	struct l2cap_channel *chan;
	struct l2cap_req *rq;
	int v = 0, err, i;
	u_int spin;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	if (r3_td == NULL) {
		err = lwkt_create(r3_rtx_thread, NULL, &r3_td, NULL,
		    0, 1 /* cpu 1 */, "l2r3rtx");
		if (err != 0) {
			kprintf("[l2uh] lwkt_create failed: %d\n", err);
			return 0;
		}
		kprintf("[l2uh] rtx kthread pinned to cpu1 (tid %p)\n", r3_td);
	}

	kprintf("[l2uh] === DF-5749 race3: %d simultaneous "
	    "l2cap_rtx() vs l2cap_detach() rounds ===\n", v);

	for (i = 0; i < v; i++) {
		chan = arm_round();
		if (chan == NULL)
			continue;
		rq = TAILQ_FIRST(&chan->lc_link->hl_reqs);

		r3_done = 0;
		r3_chan = chan;
		r3_req = rq;
		cpu_ccfence();
		atomic_add_long(&r3_seq, 1);	/* GO */

		/* the close(2) path, same instant as the rtx side */
		l2cap_detach(&chan);			/* production */
		n_detached++;

		/* wait for the rtx side to finish this round (bounded) */
		spin = 0;
		while (r3_done == 0) {
			if (++spin > 2000000000U)
				break;			/* kthread descheduled */
			cpu_ccfence();
		}

		r3_chan = NULL;
		r3_req = NULL;

		if ((i & 0x3fff) == 0) {
			kprintf("[l2uh] round %d: armed=%lu detached=%lu "
			    "disconnected=%d\n", i, n_armed, n_detached,
			    stub_proto_disconnected_count);
			/* give the 1-tick real callouts a moment too */
			tsleep(&i, 0, "l2r3p", 1);
		}
	}

	kprintf("[l2uh] RACE3 RUN COMPLETE - no panic (armed=%lu "
	    "detached=%lu disconnected=%d)\n", n_armed, n_detached,
	    stub_proto_disconnected_count);
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, l2uh_race3, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_race3, "I", "DF-5749: simultaneous rtx-vs-detach, 2 cpus");

/* ------------------------------------------------------------------ */
/*
 * DF-5749 race4: 100% production callout delivery, cross-cpu.
 *
 * An armer kthread pinned to cpu0 builds channels exactly as
 * l2cap_connect() does and arms the production 1-tick rtx callouts
 * (they fire on cpu0's softclock thread, as in production).  A reaper
 * kthread pinned to cpu1 continuously executes the production
 * close(2) teardown (l2cap_detach) on the channels as fast as they
 * are handed over.  Every channel therefore crosses its rtx deadline
 * while another cpu is tearing it down -- the exact production
 * topology (softclock vs netisr protocol port), no direct calls.
 *
 * On the unfixed kernel this panics (double close / double free);
 * on the fixed kernel it must run clean.  NOTE: race3 (direct
 * l2cap_rtx() calls) is only a valid trigger on the UNFIXED kernel:
 * after the fix, request_free() tears the request down under the
 * callout_stop() drain, which a direct call bypasses -- production
 * always delivers l2cap_rtx() via the callout machinery and cannot
 * reach a freed request that way.
 */
#define R4RING	4096
static struct l2cap_channel *volatile r4_ring[R4RING];
static volatile u_long r4_head, r4_tail;
static volatile u_long r4_target;
static volatile u_long r4_armed_ct, r4_detached_ct;
static volatile int r4_state;		/* 0=idle 1=run 2=done */
static struct thread *r4_arm_td, *r4_reap_td;

static void
r4_armer(void *arg)
{
	struct l2cap_channel *chan;

	for (;;) {
		while (r4_state != 1)
			tsleep(&r4_state, 0, "l2r4a", hz);
		while (r4_state == 1 && r4_armed_ct < r4_target) {
			while ((r4_head - r4_tail) >= R4RING) {
				/* ring full: pace to the reaper (1 tick) */
				tsleep(&r4_tail, 0, "l2r4f", 1);
				if (r4_state != 1)
					goto out;
			}
			chan = arm_round();
			if (chan == NULL)
				continue;
			r4_ring[r4_head % R4RING] = chan;
			atomic_add_long(&r4_head, 1);
			atomic_add_long(&r4_armed_ct, 1);
		}
out:
		if (r4_state == 1) {
			r4_state = 2;		/* done arming */
			wakeup(&r4_state);
		}
	}
}

static void
r4_reaper(void *arg)
{
	struct l2cap_channel *chan;

	for (;;) {
		while (r4_tail == r4_head) {
			if (r4_state == 2 &&
			    r4_detached_ct == r4_armed_ct) {
				r4_state = 0;	/* round set complete */
				wakeup(&r4_state);
			}
			if (r4_state == 0)
				tsleep(&r4_head, 0, "l2r4r", hz);
			else
				tsleep(&r4_head, 0, "l2r4r", 1);
		}
		chan = r4_ring[r4_tail % R4RING];
		r4_ring[r4_tail % R4RING] = NULL;
		atomic_add_long(&r4_tail, 1);
		if (chan != NULL) {
			l2cap_detach(&chan);		/* production */
			atomic_add_long(&r4_detached_ct, 1);
			n_detached++;
		}
	}
}

static int
l2uh_race4(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err;
	int tries;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	if (r4_arm_td == NULL) {
		err = lwkt_create(r4_armer, NULL, &r4_arm_td, NULL,
		    0, 0 /* cpu 0: its callouts fire on cpu0 softclock */,
		    "l2r4arm");
		if (err != 0) {
			kprintf("[l2uh] armer lwkt_create failed: %d\n", err);
			return 0;
		}
		err = lwkt_create(r4_reaper, NULL, &r4_reap_td, NULL,
		    0, 1 /* cpu 1: the "other cpu" tearing down */,
		    "l2r4reap");
		if (err != 0) {
			kprintf("[l2uh] reaper lwkt_create failed: %d\n", err);
			return 0;
		}
		kprintf("[l2uh] race4 threads: armer cpu0 (softclock "
		    "delivery), reaper cpu1 (detach)\n");
	}

	kprintf("[l2uh] === DF-5749 race4: %d production-callout rounds "
	    "===\n", v);
	r4_head = r4_tail = 0;
	r4_armed_ct = r4_detached_ct = 0;
	r4_target = (u_long)v;
	r4_state = 1;
	wakeup(&r4_state);

	for (tries = 0; tries < 600; tries++) {	/* up to ~10 min */
		if (r4_state == 0)
			break;
		tsleep(&tries, 0, "l2r4w", hz);
		if ((tries % 15) == 0) {
			static u_long last_armed = 0;

			kprintf("[l2uh] race4 progress: armed=%lu "
			    "detached=%lu disconnected=%d arm_phase=%d\n",
			    r4_armed_ct, r4_detached_ct,
			    stub_proto_disconnected_count, arm_phase);
			if (r4_armed_ct == last_armed &&
			    r4_armed_ct < r4_target) {
				arm_stall_seen++;
				kprintf("[l2uh] *** ARMED COUNTER STALLED "
				    "at %lu, arm_phase=%d ***\n",
				    r4_armed_ct, arm_phase);
			}
			last_armed = r4_armed_ct;
		}
	}

	kprintf("[l2uh] RACE4 %s: armed=%lu detached=%lu "
	    "disconnected=%d\n",
	    (r4_state == 0 ? "RUN COMPLETE - no panic" : "TIMEOUT"),
	    r4_armed_ct, r4_detached_ct, stub_proto_disconnected_count);
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, l2uh_race4, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_race4, "I", "DF-5749: production-callout race (2 cpus)");

/* ------------------------------------------------------------------ */
/*
 * DF-5749 race5: the PRODUCTION race pair and nothing else.
 *
 * race3 delivered l2cap_rtx() by direct call (unfixed-kernel trigger);
 * race4 additionally raced the active-list ops (cid_alloc vs detach),
 * which production serializes on the cpu0 netisr port -- a rig
 * artifact.  race5 models production exactly:
 *
 *   - ONE worker thread (cpu1) plays the netisr protocol port: it
 *     performs every list-mutating production call (l2cap_attach,
 *     l2cap_cid_alloc, l2cap_request_alloc, l2cap_detach) so those
 *     cannot race each other, exactly as the single protocol port
 *     serializes connect(2)/close(2) in production.
 *   - a cpu0 helper ONLY re-arms the production rtx callout to 1 tick
 *     (the hci_acl_close() re-arm pattern): it fires on cpu0's
 *     softclock thread, the real delivery context.
 *   - the worker sleeps 1 tick and then detaches -- landing on the
 *     same tick boundary where softclock executes l2cap_rtx().
 *
 * That is precisely: softclock l2cap_rtx() racing the netisr-port
 * l2cap_detach(), the DF-5749 interleaving.
 */
static struct l2cap_req *volatile r5_req;
static volatile u_long r5_seq, r5_acked;
static volatile u_long r5_rounds, r5_target;
static volatile int r5_state;		/* 0 idle, 1 run, 2 done */
static struct thread *r5_arm_td, *r5_work_td;

static void
r5_armer(void *arg)
{
	u_long seen = 0;
	struct l2cap_req *volatile r;

	for (;;) {
		while (r5_seq == seen) {
			if (r5_state == 0 || r5_state == 2) {
				tsleep(&r5_seq, 0, "l2r5a", 1);
				continue;
			}
			tsleep(&r5_seq, 0, "l2r5a", 1);
		}
		seen = r5_seq;
		r = r5_req;
		if (r != NULL)
			callout_reset(&r->lr_rtx, 1, l2cap_rtx, r);
		atomic_add_long(&r5_acked, 1);
	}
}

static void
r5_worker(void *arg)
{
	struct l2cap_channel *chan;
	u_long n = 0;

	for (;;) {
		while (r5_state != 1)
			tsleep(&r5_state, 0, "l2r5w", hz);
		while (r5_state == 1 && n < r5_target) {
			chan = arm_round();
			if (chan == NULL)
				continue;
			r5_req = TAILQ_FIRST(&chan->lc_link->hl_reqs);
			atomic_add_long(&r5_seq, 1);	/* GO (cpu0 arms) */
			while (r5_acked <= n)
			tsleep(&r5_acked, 0, "l2r5k", 1);
			/* land on the tick where the rtx fires */
			tsleep(&n, 0, "l2r5t", 1);
			l2cap_detach(&chan);		/* production */
			n_detached++;
			n++;
			atomic_add_long(&r5_rounds, 1);
		}
		if (r5_state == 1) {
			r5_state = 2;
			wakeup(&r5_state);
		}
	}
}

static int
l2uh_race5(SYSCTL_HANDLER_ARGS)
{
	int v = 0, err, tries;

	err = sysctl_handle_int(oidp, &v, 0, req);
	if (err != 0 || req->newptr == NULL || v == 0)
		return err;

	if (r5_work_td == NULL) {
		err = lwkt_create(r5_armer, NULL, &r5_arm_td, NULL,
		    0, 0, "l2r5arm");
		if (err == 0)
			err = lwkt_create(r5_worker, NULL, &r5_work_td,
			    NULL, 0, 1, "l2r5work");
		if (err != 0) {
			kprintf("[l2uh] race5 lwkt_create failed: %d\n", err);
			return 0;
		}
		kprintf("[l2uh] race5: callout armer cpu0 (softclock "
		    "delivery), worker cpu1 (all port ops + detach)\n");
	}

	kprintf("[l2uh] === DF-5749 race5: %d production-topology rounds "
	    "===\n", v);
	r5_seq = r5_acked = 0;
	r5_rounds = 0;
	r5_target = (u_long)v;
	r5_state = 1;
	wakeup(&r5_state);
	wakeup(&r5_seq);

	for (tries = 0; tries < 900; tries++) {
		if (r5_state == 2)
			break;
		tsleep(&tries, 0, "l2r5c", hz);
		if ((tries % 20) == 0)
			kprintf("[l2uh] race5 progress: rounds=%lu "
			    "disconnected=%d\n", r5_rounds,
			    stub_proto_disconnected_count);
	}

	kprintf("[l2uh] RACE5 %s: rounds=%lu disconnected=%d\n",
	    (r5_state == 2 ? "RUN COMPLETE - no panic" : "TIMEOUT"),
	    r5_rounds, stub_proto_disconnected_count);
	r5_state = 0;
	return 0;
}

SYSCTL_PROC(_hw, OID_AUTO, l2uh_race5, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, l2uh_race5, "I", "DF-5749: production pair only (rtx vs detach)");

static int
harness_mode(module_t mod, int type, void *data)
{
	switch (type) {
	case MOD_LOAD:
		l2cap_pdu_pool = zinit("l2cap_pdu",
		    sizeof(struct l2cap_pdu), 1, ZONE_DESTROYABLE);
		l2cap_req_pool = zinit("l2cap_req",
		    sizeof(struct l2cap_req), 1, ZONE_DESTROYABLE);
		KKASSERT(l2cap_pdu_pool != NULL && l2cap_req_pool != NULL);
		kprintf("[l2uh] l2uh_harness loaded: hw.l2uh_setup, "
		    "hw.l2uh_once, hw.l2uh_race, hw.l2uh_race3, hw.l2uh_state\n");
		break;
	case MOD_UNLOAD:
		return EBUSY;
	default:
		break;
	}
	return 0;
}

static moduledata_t harness_moddata = {
	"l2uh_harness",
	harness_mode,
	NULL
};

DECLARE_MODULE(l2uh_harness, harness_moddata, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);