/*
 * fwpci_harness.c -- verification harness for DF-2304 / DF-2305 /
 * DF-2306.  Audit target: sys/bus/firewire/fwohci_pci.c.
 *
 * VERBATIM-INTERPOSITION DESIGN (extends the DF-2293 fwdev.c and
 * DF-2300..2303 fwohci.c rigs):
 *   - This translation unit #includes the UNMODIFIED
 *     sys/bus/firewire/fwohci_pci.c (the stock kernel links
 *     firewire.o/fwdev.o/fwdma.o/fwmem.o but fwohci_pci.c is disabled
 *     in conf/files, so no duplicate symbols).  Its statics become
 *     callable by the rig:
 *       fwohci_pci_probe   (fwohci_pci.c:71)    -- DF-2306
 *       fwohci_pci_attach  (fwohci_pci.c:249)   -- DF-2305, DF-2304 entry
 *       fwohci_pci_add_child (fwohci_pci.c:428) -- DF-2304, reached via
 *         the REAL bus_generic_probe -> DEVICE_IDENTIFY ->
 *         BUS_ADD_CHILD dispatch chain from inside the real attach
 *       fwohci_pci_detach  (fwohci_pci.c:342)
 *   - Three call sites inside the verbatim file are interposed with
 *     macros defined BEFORE the #include (the wrapper bodies are
 *     parsed before the macros exist, so their internal calls hit the
 *     REAL kernel functions):
 *       device_probe_and_attach  (fwohci_pci.c:442) -- forced-failure
 *         interposition for the DF-2304 latent error branch
 *       bus_dma_tag_create       (fwohci_pci.c:300) -- forced-failure
 *         interposition for the DF-2305 error branch
 *       device_delete_child      (fwohci_pci.c:352 and :447) -- pure
 *         logging wrapper (forwards to the REAL device_delete_child)
 *   - The OHCI-controller externalities are the ones already provided
 *     by the fwohci_harness.c rig (emulated MMIO register file
 *     fh_regs[] + emulated PHY behind OHCI_PHYACCESS, 1 ms callout);
 *     fh_regs is exported (un-staticed) by the extended copy of that
 *     harness installed by fwpci_prep.sh.
 *   - The PCI externalities are fabricated with a real kobj bus class
 *     ("fwp_pcibus") implementing exactly the bus(9)/pci(9) methods
 *     fwohci_pci.c dispatches through its parent: bus_read_ivar,
 *     bus_alloc_resource / release_resource, bus_setup_intr /
 *     teardown_intr (with DF-2305 resource accounting + a rman-style
 *     BAR-conflict model so a leaked outstanding BAR makes a second
 *     attach fail, exactly as the real pci rman would), and
 *     pci_read_config / pci_write_config against an emulated config
 *     space.  The fake fwohci device is a struct bsd_device shell with
 *     a real kobj class whose bus_add_child method is the verbatim
 *     fwohci_pci_add_child, parent = the fabricated pcibus, devclass =
 *     the real "fwohci" devclass (created by the verbatim
 *     DRIVER_MODULE registration at SI_SUB_DRIVERS), so the REAL
 *     bus_generic_probe inside the REAL fwohci_pci_attach dispatches
 *     the REAL firewire identify -> BUS_ADD_CHILD ->
 *     fwohci_pci_add_child chain, which calls the REAL
 *     device_add_child("firewire") creating a REAL new-bus child in
 *     the real firewire devclass and on the global bus_data_devices
 *     list.  Everything after that is stock kernel code.
 *   - /dev/fwpcictl0 (root-only, 0600): rig control (set fabricated
 *     PCI identity + run real probe; run real attach with per-call
 *     forced failures; resource-accounting dump; forced cleanup
 *     detach; bootverbose toggle for the DF-2306 A/B).
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/conf.h>
#include <sys/device.h>
#include <sys/bus.h>
#include <sys/bus_private.h>
#include <sys/queue.h>
#include <sys/rman.h>
#include <sys/callout.h>
#include <sys/sysctl.h>
#include <sys/thread2.h>

/* register offsets re-declared locally: OHCI_VERSION/OHCI_BUS_OPT
 * are #defines inside fwohci.c (fwohci.c:153/156), which this TU does
 * not include (fwohci_pci.c is the verbatim target here) */
#define FWP_OHCI_VERSION	0x00	/* fwohci.c:153 */
#define FWP_OHCI_BUS_OPT	0x20	/* fwohci.c:156 */

#include <bus/pci/pcireg.h>
#include <bus/pci/pcivar.h>	/* also pulls pci_if.h method macros */

/* exported by the extended fwohci_harness.c rig (fwpci_prep.sh) */
extern u_int32_t fh_regs[];	/* emulated OHCI MMIO register file */

MALLOC_DEFINE(M_FWPCI, "fwpci_harness", "fwohci_pci.c verification harness");

/* ------------------------------------------------------------------ */
/* interposition wrappers (defined BEFORE the renaming #defines)       */
/* ------------------------------------------------------------------ */
static int fwp_dpa_arm;		/* fail the Nth device_probe_and_attach */
static int fwp_dpa_fail_at = 1;
static u_int fwp_dpa_calls;

static int fwp_dmatag_arm;	/* fail the Nth bus_dma_tag_create */
static int fwp_dmatag_fail_at = 1;
static u_int fwp_dmatag_calls;

struct fwp_delog {
	device_t	parent;
	device_t	child;
	int		line;	/* call site __LINE__ inside fwohci_pci.c */
	int		rc;
	u_int64_t	when;	/* ticks at call time */
};
#define FWP_DELOG_MAX 8
static struct fwp_delog fwp_delog[FWP_DELOG_MAX];
static u_int fwp_delog_n;

static int
fwp_device_probe_and_attach(device_t dev)
{
	fwp_dpa_calls++;
	if (fwp_dpa_arm && (int)fwp_dpa_calls >= fwp_dpa_fail_at) {
		fwp_dpa_arm = 0;
		kprintf("FWP: device_probe_and_attach(%p) INTERPOSED FAIL "
		    "(ENXIO) call#%u  [DF-2304 forced branch]\n",
		    dev, fwp_dpa_calls);
		return (ENXIO);
	}
	return (device_probe_and_attach(dev));	/* the REAL one */
}

static int
fwp_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
		       bus_size_t boundary, bus_addr_t lowaddr,
		       bus_addr_t highaddr, bus_size_t maxsize, int nsegments,
		       bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat)
{
	fwp_dmatag_calls++;
	if (fwp_dmatag_arm && (int)fwp_dmatag_calls >= fwp_dmatag_fail_at) {
		fwp_dmatag_arm = 0;
		*dmat = NULL;
		kprintf("FWP: bus_dma_tag_create() INTERPOSED FAIL (ENOMEM) "
		    "call#%u  [DF-2305 forced branch]\n", fwp_dmatag_calls);
		return (ENOMEM);
	}
	return (bus_dma_tag_create(parent, alignment, boundary, lowaddr,	/* REAL */
	    highaddr, maxsize, nsegments, maxsegsz, flags, dmat));
}

static int
fwp_device_delete_child(device_t dev, device_t child, int line)
{
	int rc;

	kprintf("FWP: device_delete_child(dev=%p child=%p) FROM "
	    "fwohci_pci.c:%d\n", dev, child, line);
	rc = device_delete_child(dev, child);	/* the REAL one */
	if (fwp_delog_n < FWP_DELOG_MAX) {
		fwp_delog[fwp_delog_n].parent = dev;
		fwp_delog[fwp_delog_n].child = child;
		fwp_delog[fwp_delog_n].line = line;
		fwp_delog[fwp_delog_n].rc = rc;
		fwp_delog[fwp_delog_n].when = ticks;
	}
	fwp_delog_n++;
	kprintf("FWP: device_delete_child(child=%p) returned %d\n",
	    child, rc);
	return (rc);
}

/* ------------------------------------------------------------------ */
/* the audited file, verbatim                                          */
/* ------------------------------------------------------------------ */
#define device_probe_and_attach(dev)	fwp_device_probe_and_attach(dev)
#define bus_dma_tag_create		fwp_bus_dma_tag_create
#define device_delete_child(dev, child)	\
	fwp_device_delete_child((dev), (child), __LINE__)
#include "fwohci_pci.c"
#undef device_probe_and_attach
#undef bus_dma_tag_create
#undef device_delete_child

/* fwohci_init/fwohci_stop/fwohci_detach/fwohci_intr/fwohci_poll come
 * from the fwohci_harness.c TU (fwohci.c compiled verbatim there) */

/* ------------------------------------------------------------------ */
/* fabricated PCI config space / identity                              */
/* ------------------------------------------------------------------ */
struct fwp_pcicfg {
	u_int16_t	vendor;
	u_int16_t	device;
	u_int16_t	command;
	u_int8_t	class;
	u_int8_t	subclass;
	u_int8_t	progif;
	u_int8_t	revid;
	u_int8_t	intpin;
	u_int8_t	intline;
	u_int8_t	lattimer;
	u_int8_t	cacheline;
};
static struct fwp_pcicfg fwp_cfg = {
	.vendor = 0x1a2b,		/* NOT in fwohci_pci_probe ID table */
	.device = 0x3c4d,
	.command = 0x0007,
	.class = PCIC_SERIALBUS,	/* 0x0c */
	.subclass = PCIS_SERIALBUS_FW,	/* 0x00 */
	.progif = PCI_INTERFACE_OHCI,	/* 0x10 -> generic-class fallthrough */
	.revid = 0x00,
	.intpin = 1,
	.intline = 11,			/* valid: not 0, not 255 */
	.lattimer = 0x10,
	.cacheline = 0x00,
};

/* ------------------------------------------------------------------ */
/* fabricated bus(9)/pci(9) methods for the fake parent PCI bus        */
/* ------------------------------------------------------------------ */

/* DF-2305 resource accounting (the proof instrument) */
static u_int fwp_mem_allocs, fwp_mem_releases;
static u_int fwp_irq_allocs, fwp_irq_releases;
static u_int fwp_intr_setups, fwp_intr_teardowns, fwp_live_handlers;
static struct resource *fwp_mem_outstanding;	/* rman BAR model */
static struct resource *fwp_irq_outstanding;
static int fwp_irq_fail_arm;			/* sibling-path interposer */
static u_int fwp_mem_conflict_fails;		/* re-attach failure proof */

static struct resource *
fwp_bus_alloc_resource(device_t dev, device_t child, int type, int *rid,
    u_long start, u_long end, u_long count, u_int flags, int cpuid)
{
	struct resource *r;

	if (type == SYS_RES_MEMORY) {
		if (fwp_mem_outstanding != NULL) {
			/* rman semantics: the BAR range is still owned by
			 * the leaked allocation -> NULL, exactly as
			 * pci_alloc_resource/rman would refuse it */
			fwp_mem_conflict_fails++;
			kprintf("FWP: bus_alloc_resource(MEM) -> NULL "
			    "(BAR 0x%lx still outstanding: rman conflict)\n",
			    fwp_mem_outstanding->r_start);
			return (NULL);
		}
		r = kmalloc(sizeof(*r), M_FWPCI, M_INTWAIT | M_ZERO);
		r->r_start = 0xF0000000ul;
		r->r_end = 0xF0007FFFul;
		r->r_rid = *rid;
		r->r_dev = child;
		rman_set_bustag(r, X86_64_BUS_SPACE_MEM);
		rman_set_bushandle(r, (bus_space_handle_t)&fh_regs[0]);
		fwp_mem_outstanding = r;
		fwp_mem_allocs++;
		kprintf("FWP: bus_alloc_resource(MEM rid=%d) -> %p "
		    "(allocs=%u)\n", *rid, r, fwp_mem_allocs);
		return (r);
	}
	if (type == SYS_RES_IRQ) {
		if (fwp_irq_fail_arm) {
			fwp_irq_fail_arm = 0;
			kprintf("FWP: bus_alloc_resource(IRQ) -> NULL "
			    "(FORCED; models sibling error path "
			    "fwohci_pci.c:280)\n");
			return (NULL);
		}
		r = kmalloc(sizeof(*r), M_FWPCI, M_INTWAIT | M_ZERO);
		r->r_start = r->r_end = fwp_cfg.intline;
		r->r_rid = *rid;
		r->r_dev = child;
		fwp_irq_outstanding = r;
		fwp_irq_allocs++;
		kprintf("FWP: bus_alloc_resource(IRQ rid=%d) -> %p "
		    "(allocs=%u)\n", *rid, r, fwp_irq_allocs);
		return (r);
	}
	return (NULL);
}

static int
fwp_bus_activate_resource(device_t dev, device_t child, int type, int rid,
    struct resource *r)
{
	return (0);
}

static int
fwp_bus_deactivate_resource(device_t dev, device_t child, int type, int rid,
    struct resource *r)
{
	return (0);
}

static int
fwp_bus_release_resource(device_t dev, device_t child, int type, int rid,
    struct resource *r)
{
	if (type == SYS_RES_MEMORY && r == fwp_mem_outstanding) {
		fwp_mem_outstanding = NULL;
		fwp_mem_releases++;
		kfree(r, M_FWPCI);
		kprintf("FWP: bus_release_resource(MEM) ok "
		    "(releases=%u)\n", fwp_mem_releases);
		return (0);
	}
	if (type == SYS_RES_IRQ && r == fwp_irq_outstanding) {
		fwp_irq_outstanding = NULL;
		fwp_irq_releases++;
		kfree(r, M_FWPCI);
		kprintf("FWP: bus_release_resource(IRQ) ok "
		    "(releases=%u)\n", fwp_irq_releases);
		return (0);
	}
	kprintf("FWP: bus_release_resource(type=%d r=%p) -> EINVAL "
	    "(unknown resource)\n", type, r);
	return (EINVAL);
}

static void *fwp_intr_cookie = (void *)0xF00DCAFE;

static int
fwp_bus_setup_intr(device_t dev, device_t child, struct resource *irq,
    int flags, driver_intr_t *intr, void *arg, void **cookiep,
    lwkt_serialize_t serializer, const char *desc)
{
	fwp_intr_setups++;
	fwp_live_handlers++;
	*cookiep = fwp_intr_cookie;
	kprintf("FWP: bus_setup_intr(handler=%p arg=%p) REGISTERED "
	    "(setups=%u live=%u)\n", intr, arg, fwp_intr_setups,
	    fwp_live_handlers);
	return (0);
}

static int
fwp_bus_teardown_intr(device_t dev, device_t child, struct resource *irq,
    void *cookie)
{
	if (cookie != fwp_intr_cookie)
		return (EINVAL);
	fwp_intr_teardowns++;
	if (fwp_live_handlers)
		fwp_live_handlers--;
	kprintf("FWP: bus_teardown_intr(cookie=%p) ok (teardowns=%u "
	    "live=%u)\n", cookie, fwp_intr_teardowns, fwp_live_handlers);
	return (0);
}

static int
fwp_bus_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
{
	switch (index) {
	case PCI_IVAR_VENDOR:	*result = fwp_cfg.vendor; return (0);
	case PCI_IVAR_DEVICE:	*result = fwp_cfg.device; return (0);
	case PCI_IVAR_SUBVENDOR:*result = 0x0000; return (0);
	case PCI_IVAR_SUBDEVICE:*result = 0x0000; return (0);
	case PCI_IVAR_CLASS:	*result = fwp_cfg.class; return (0);
	case PCI_IVAR_SUBCLASS:	*result = fwp_cfg.subclass; return (0);
	case PCI_IVAR_PROGIF:	*result = fwp_cfg.progif; return (0);
	case PCI_IVAR_REVID:	*result = fwp_cfg.revid; return (0);
	case PCI_IVAR_INTPIN:	*result = fwp_cfg.intpin; return (0);
	case PCI_IVAR_IRQ:	*result = fwp_cfg.intline; return (0);
	case PCI_IVAR_DOMAIN:	*result = 0; return (0);
	case PCI_IVAR_BUS:	*result = 0; return (0);
	case PCI_IVAR_SLOT:	*result = 5; return (0);
	case PCI_IVAR_FUNCTION:	*result = 0; return (0);
	}
	return (ENOENT);
}

static int
fwp_bus_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
{
	return (ENOENT);
}

static u_int32_t
fwp_pci_read_config(device_t dev, device_t child, int reg, int width)
{
	u_int32_t v = 0;

	switch (reg) {
	case PCIR_COMMAND:	v = fwp_cfg.command; break;
	case PCIR_CACHELNSZ:	v = fwp_cfg.cacheline; break;
	case PCIR_LATTIMER:	v = fwp_cfg.lattimer; break;
	case PCIR_INTLINE:	v = fwp_cfg.intline; break;
	case PCIR_INTPIN:	v = fwp_cfg.intpin; break;
	case PCIR_REVID:	v = fwp_cfg.revid; break;
	default:
		v = 0;
		break;
	}
	if (width == 1)
		v &= 0xff;
	else if (width == 2)
		v &= 0xffff;
	return (v);
}

static void
fwp_pci_write_config(device_t dev, device_t child, int reg, u_int32_t val,
    int width)
{
	u_int32_t mask = (width == 1) ? 0xff :
	    (width == 2) ? 0xffff : 0xffffffff;

	switch (reg) {
	case PCIR_COMMAND:
		fwp_cfg.command =
		    (fwp_cfg.command & ~mask) | (u_int16_t)(val & mask);
		break;
	case PCIR_CACHELNSZ:	fwp_cfg.cacheline = (u_int8_t)val; break;
	case PCIR_LATTIMER:	fwp_cfg.lattimer = (u_int8_t)val; break;
	case PCIR_INTLINE:	fwp_cfg.intline = (u_int8_t)val; break;
	}
}

/* ------------------------------------------------------------------ */
/* fabricated kobj classes + device shells                             */
/* ------------------------------------------------------------------ */
static struct kobj_method fwp_pcibus_methods[] = {
	KOBJMETHOD(bus_read_ivar,		fwp_bus_read_ivar),
	KOBJMETHOD(bus_write_ivar,		fwp_bus_write_ivar),
	KOBJMETHOD(bus_alloc_resource,		fwp_bus_alloc_resource),
	KOBJMETHOD(bus_activate_resource,	fwp_bus_activate_resource),
	KOBJMETHOD(bus_deactivate_resource,	fwp_bus_deactivate_resource),
	KOBJMETHOD(bus_release_resource,	fwp_bus_release_resource),
	KOBJMETHOD(bus_setup_intr,		fwp_bus_setup_intr),
	KOBJMETHOD(bus_teardown_intr,		fwp_bus_teardown_intr),
	KOBJMETHOD(pci_read_config,		fwp_pci_read_config),
	KOBJMETHOD(pci_write_config,		fwp_pci_write_config),
	KOBJMETHOD_END
};
static struct kobj_class fwp_pcibus_class = {
	"fwp_pcibus", fwp_pcibus_methods, 0
};

/* the fake fwohci device: its bus_add_child method IS the verbatim
 * fwohci_pci_add_child (mirrors fwohci_methods, fwohci_pci.c:473) */
static struct kobj_method fwp_fwbus_methods[] = {
	KOBJMETHOD(bus_add_child,		fwohci_pci_add_child),
	KOBJMETHOD(bus_print_child,		bus_generic_print_child),
	KOBJMETHOD_END
};
static struct kobj_class fwp_fwbus_class = {
	"fwp_fwbus", fwp_fwbus_methods, 0
};

static struct bsd_device fwp_pcibus_dev;	/* fake PCI bus */
static struct bsd_device fwp_fw_dev;		/* fake fwohci PCI device */
static struct bsd_device fwp_probe_dev;		/* DF-2306 probe shell */
static fwohci_softc_t *fwp_sc;			/* the fake device softc */
static char fwp_fw_nameunit[] = "fwohci0";
static char fwp_probe_nameunit[] = "fwohci6";
static char fwp_pcibus_nameunit[] = "fwp_pcibus0";

static struct dev_ops fwpci_ops;

/* xferq fabrication (same recipe as the DF-2300 rig) */
static struct fw_xferq *
fwp_newq(int dmach)
{
	struct fw_xferq *q = kmalloc(sizeof(*q), M_FWPCI, M_INTWAIT | M_ZERO);

	q->dmach = dmach;
	q->maxq = FWMAXQUEUE;
	STAILQ_INIT(&q->q);
	STAILQ_INIT(&q->binds);
	STAILQ_INIT(&q->stvalid);
	STAILQ_INIT(&q->stfree);
	STAILQ_INIT(&q->stdma);
	return (q);
}

/* per-run softc preparation: zeroed softc + fabricated fc fields +
 * stock boot-time OHCI register values (fwohci_init mutates them).
 *
 * Rig note: resource-pointer fields that reference still-outstanding
 * rig resources are PRESERVED across the wipe, modelling the real
 * window where the leaked BAR/IRQ/handler are still reachable through
 * the (not yet freed) softc -- so a rig forced-detach can release
 * exactly what the stock DF-2305 error path leaves behind. */
static void
fwp_prep_softc(void)
{
	struct resource *saved_bsr = fwp_sc->bsr;
	struct resource *saved_irq = fwp_sc->irq_res;
	void *saved_ih = fwp_sc->ih;
	bus_space_tag_t saved_bst = fwp_sc->bst;
	bus_space_handle_t saved_bsh = fwp_sc->bsh;
	int i;

	memset(fwp_sc, 0, sizeof(*fwp_sc));
	if (fwp_mem_outstanding != NULL && saved_bsr == fwp_mem_outstanding) {
		fwp_sc->bsr = saved_bsr;
		fwp_sc->bst = saved_bst;
		fwp_sc->bsh = saved_bsh;
	}
	if (fwp_irq_outstanding != NULL && saved_irq == fwp_irq_outstanding)
		fwp_sc->irq_res = saved_irq;
	if (fwp_live_handlers != 0 && saved_ih == fwp_intr_cookie)
		fwp_sc->ih = saved_ih;
	fwp_sc->fc.eui.hi = 0x00112233;
	fwp_sc->fc.eui.lo = 0x44556677;
	fwp_sc->fc.maxrec = 10;
	fwp_sc->fc.max_asyretry = 4;
	fwp_sc->fc.status = FWBUSCOMPLETION;
	fwp_sc->fc.nisodma = 4;
	fwp_sc->fc.dev = (device_t)&fwp_fw_dev;

	STAILQ_INIT(&fwp_sc->fc.binds);
	STAILQ_INIT(&fwp_sc->fc.devices);
	for (i = 0; i < 0x40; i++)
		STAILQ_INIT(&fwp_sc->fc.tlabels[i]);
	for (i = 0; i < 4; i++) {
		fwp_sc->fc.ir[i] = fwp_newq(i);
		fwp_sc->fc.it[i] = fwp_newq(i);
	}
	fwp_sc->fc.arq = fwp_newq(-1);
	fwp_sc->fc.ars = fwp_newq(-1);
	fwp_sc->fc.atq = fwp_newq(-1);
	fwp_sc->fc.ats = fwp_newq(-1);

	/* stock boot register values: OHCI version 1.4 (mver bits 16-23
	 * must be 1..9 for fwohci_init, fwohci.c:592-600); IT/IR iso
	 * channel masks have plain-store semantics in this emulation, so
	 * fwohci_init's 0xffffffff probe readback yields nisodma=32
	 * (within the it[0x20]/ir[0x20] softc arrays, fwohcivar.h:64) */
	fh_regs[FWP_OHCI_VERSION / 4] = 0x00010400;
	fh_regs[FWP_OHCI_BUS_OPT / 4] = 0xf << 12;
}

static void
fwp_init(void *arg)
{
	fwp_sc = kmalloc(sizeof(*fwp_sc), M_FWPCI, M_INTWAIT | M_ZERO);

	/* fake PCI bus shell */
	TAILQ_INIT(&fwp_pcibus_dev.children);
	fwp_pcibus_dev.nameunit = fwp_pcibus_nameunit;
	fwp_pcibus_dev.state = DS_ATTACHED;
	kobj_init((kobj_t)&fwp_pcibus_dev, &fwp_pcibus_class);

	/* fake fwohci PCI device shell (state DS_ALIVE mirrors
	 * device_probe_child before the synchronous device_doattach) */
	TAILQ_INIT(&fwp_fw_dev.children);
	fwp_fw_dev.nameunit = fwp_fw_nameunit;
	fwp_fw_dev.parent = &fwp_pcibus_dev;
	fwp_fw_dev.devclass = devclass_find("fwohci");
	fwp_fw_dev.state = DS_ALIVE;
	fwp_fw_dev.softc = fwp_sc;
	fwp_fw_dev.flags = DF_ENABLED;
	kobj_init((kobj_t)&fwp_fw_dev, &fwp_fwbus_class);

	/* DF-2306 probe shell: device_printf -> "fwohci6: " */
	fwp_probe_dev.nameunit = fwp_probe_nameunit;
	fwp_probe_dev.parent = &fwp_pcibus_dev;
	fwp_probe_dev.devclass = fwp_fw_dev.devclass;
	fwp_probe_dev.unit = 6;
	fwp_probe_dev.state = DS_NOTPRESENT;

	fwp_prep_softc();

	make_dev(&fwpci_ops, 0, UID_ROOT, GID_WHEEL, 0600, "fwpcictl0");

	kprintf("FWPCIHARNESS-READY: fwohci_pci.c verbatim (TU-compiled), "
	    "fake pcibus=%p fwohci-dev=%p sc=%p regs=%p "
	    "fwohci-devclass=%p /dev/fwpcictl0 (0600 root)\n",
	    &fwp_pcibus_dev, &fwp_fw_dev, fwp_sc, fh_regs,
	    fwp_fw_dev.devclass);
	if (fwp_fw_dev.devclass == NULL)
		kprintf("FWP: WARNING fwohci devclass missing!\n");
}
SYSINIT(fwpci_harness, SI_SUB_PSEUDO, SI_ORDER_LATER, fwp_init, NULL);

/* ------------------------------------------------------------------ */
/* /dev/fwpcictl0: root-only rig control                               */
/* ------------------------------------------------------------------ */
/* bootverbose: already declared via sys headers (defined in
 * kern/kern_shutdown.c:143) */

#include "fwpci_harness.h"

static int fwpci_open(struct dev_open_args *ap);

static int
fwpci_ioctl(struct dev_ioctl_args *ap)
{
	struct fwp_attachreq *ar;
	struct fwp_probereq *pr;
	struct fwp_ctlreq *cr;
	u_int i, m0, m1, i0, i1, s0, t0, l0, c0;
	int err = 0, rc;

	switch (ap->a_cmd) {
	case FWPCI_IOC_ATTACH:
		ar = (struct fwp_attachreq *)ap->a_data;
		fwp_prep_softc();
		fwp_fw_dev.state = DS_ALIVE;	/* re-arm each run */
		fwp_dpa_calls = 0;
		fwp_dmatag_calls = 0;
		fwp_dpa_arm = (ar->mode & FWP_MODE_DPAFAIL) ? 1 : 0;
		fwp_dpa_fail_at = 1;
		fwp_dmatag_arm = (ar->mode & FWP_MODE_DMATAGFAIL) ? 1 : 0;
		fwp_dmatag_fail_at = 1;
		fwp_irq_fail_arm = (ar->mode & FWP_MODE_IRQFAIL) ? 1 : 0;
		fwp_delog_n = 0;
		memset(fwp_delog, 0, sizeof(fwp_delog));
		m0 = fwp_mem_allocs; m1 = fwp_mem_releases;
		i0 = fwp_irq_allocs; i1 = fwp_irq_releases;
		s0 = fwp_intr_setups; t0 = fwp_intr_teardowns;
		l0 = fwp_live_handlers; c0 = fwp_mem_conflict_fails;

		kprintf("FWP: ATTACH mode=0x%x -> REAL fwohci_pci_attach"
		    "(%p)\n", ar->mode, &fwp_fw_dev);
		rc = fwohci_pci_attach(&fwp_fw_dev);	/* THE REAL ONE */
		kprintf("FWP: ATTACH returned %d\n", rc);

		ar->rc = (u_int32_t)rc;
		ar->mem_allocs = fwp_mem_allocs - m0;
		ar->mem_releases = fwp_mem_releases - m1;
		ar->irq_allocs = fwp_irq_allocs - i0;
		ar->irq_releases = fwp_irq_releases - i1;
		ar->intr_setups = fwp_intr_setups - s0;
		ar->intr_teardowns = fwp_intr_teardowns - t0;
		ar->live_handlers = fwp_live_handlers;
		ar->mem_conflict_fails = fwp_mem_conflict_fails - c0;
		ar->dpa_calls = fwp_dpa_calls;
		ar->dmatag_calls = fwp_dmatag_calls;
		ar->dellog_n = fwp_delog_n;
		for (i = 0; i < FWP_DELOG_MAX; i++) {
			ar->del_lines[i] = (u_int32_t)fwp_delog[i].line;
			ar->del_childs[i] = (u_int64_t)(uintptr_t)
			    fwp_delog[i].child;
			ar->del_rcs[i] = (u_int32_t)fwp_delog[i].rc;
		}
		break;

	case FWPCI_IOC_PROBE:
		pr = (struct fwp_probereq *)ap->a_data;
		fwp_cfg.vendor = pr->vendor;
		fwp_cfg.device = pr->device;
		fwp_cfg.class = pr->class;
		fwp_cfg.subclass = pr->subclass;
		fwp_cfg.progif = pr->progif;
		kprintf("FWP: PROBE vendor=%04x device=%04x class=%02x "
		    "subclass=%02x progif=%02x -> REAL fwohci_pci_probe"
		    "(%p)\n", pr->vendor, pr->device, pr->class,
		    pr->subclass, pr->progif, &fwp_probe_dev);
		rc = fwohci_pci_probe(&fwp_probe_dev);	/* THE REAL ONE */
		kprintf("FWP: PROBE returned %d desc=\"%s\"\n", rc,
		    fwp_probe_dev.desc ? fwp_probe_dev.desc : "(none)");
		pr->rc = (u_int32_t)rc;
		break;

	case FWPCI_IOC_CTL:
		cr = (struct fwp_ctlreq *)ap->a_data;
		switch (cr->op) {
		case 1:	/* rig hygiene: release anything still outstanding */
			kprintf("FWP: CTL forced fwohci_pci_detach (rig "
			    "cleanup; the stock code never does this on "
			    "the DF-2305 path)\n");
			rc = fwohci_pci_detach(&fwp_fw_dev);
			cr->rc = (u_int32_t)rc;
			break;
		case 2:
			bootverbose = cr->arg ? 1 : 0;
			kprintf("FWP: bootverbose=%d\n", bootverbose);
			cr->rc = 0;
			break;
		case 3:
			cr->mem_allocs = fwp_mem_allocs;
			cr->mem_releases = fwp_mem_releases;
			cr->irq_allocs = fwp_irq_allocs;
			cr->irq_releases = fwp_irq_releases;
			cr->intr_setups = fwp_intr_setups;
			cr->intr_teardowns = fwp_intr_teardowns;
			cr->live_handlers = fwp_live_handlers;
			cr->mem_conflict_fails = fwp_mem_conflict_fails;
			cr->dellog_n = fwp_delog_n;
			cr->rc = 0;
			break;
		default:
			err = EINVAL;
		}
		break;

	default:
		err = ENOTTY;
	}
	return (err);
}

static struct dev_ops fwpci_ops = {
	{ "fwpcictl", 0, D_MEM },
	.d_open =	fwpci_open,
	.d_ioctl =	fwpci_ioctl,
};

static int
fwpci_open(struct dev_open_args *ap)
{
	return (0);
}