/* * dfbt_rig.c -- production-handler dispatch rig for the DragonFly bt(4) * (BusLogic MultiMaster) driver, used to verify findings DF-2429, * DF-4097, DF-4098, DF-4099, DF-4100 and DF-4101. * * The bt driver is compiled into X86_64_GENERIC (nm shows btaction, * btdone, btreset, bttimeout, bt_intr_locked), but this QEMU guest has * no BusLogic PCI HBA, so bt_attach() never runs. This rig stands in * ONLY for the missing PCI hardware: * * - resolves the REAL static production handlers from the running * kernel image (no KASLR: file vaddr == runtime vaddr); the host * run script byte-verifies their prologues via sysctl before use; * - registers a REAL cam_sim whose sim_action/sim_poll ARE the * production btaction/btpoll, over a fabricated post-attach * bt_softc (real lockmgr lock, real bus_dma tags/maps, real DMA * memory for the CCB array and mailboxes); * - drives the REAL CAM XPT layer via xpt_action() with real * xpt_alloc_ccb'd CCBs on a real registered bus; * - plays the role of the host adapter firmware: writes completion * status into the CCB DMA memory and delivers mailbox completions * to the production btdone()/bt_intr_locked() exactly as * bt.c:1363-1370 does, and scripts a virtual 4-byte register file * (bt_inb/bt_outb are plain volatile memory accesses when the * resource tag is X86_64_BUS_SPACE_MEM, see * sys/cpu/x86_64/include/bus_dma.h:134-142). * * Everything from dev_ops dispatch through xpt_action, the XPT device * queues, btaction, btexecuteccb, btdone, bttimeout and btreset is * production code executing from the shipped kernel image. * * Root-only device /dev/dfbt0. Unload-safe on the clean-run boot. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* struct device (nameunit) */ /* ------------------------------------------------------------------ */ /* Production handler addresses, primed from userland (no KASLR). */ /* ------------------------------------------------------------------ */ static unsigned long a_btaction, a_btpoll, a_btdone, a_bttimeout, a_btreset, a_bt_intr_locked; SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btaction, CTLFLAG_RW, &a_btaction, 0, "runtime addr of production btaction"); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btpoll, CTLFLAG_RW, &a_btpoll, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btdone, CTLFLAG_RW, &a_btdone, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_bttimeout, CTLFLAG_RW, &a_bttimeout, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btreset, CTLFLAG_RW, &a_btreset, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_a_btintr, CTLFLAG_RW, &a_bt_intr_locked, 0, ""); /* first 8 bytes at each production address (host script verifies) */ static uint64_t m_btaction, m_btpoll, m_btdone, m_bttimeout, m_btreset, m_btintr; SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btaction, CTLFLAG_RD, &m_btaction, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btpoll, CTLFLAG_RD, &m_btpoll, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btdone, CTLFLAG_RD, &m_btdone, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_bttimeout, CTLFLAG_RD, &m_bttimeout, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btreset, CTLFLAG_RD, &m_btreset, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfbt_m_btintr, CTLFLAG_RD, &m_btintr, 0, ""); static int vhba_enable = 1; /* virtual HBA reset state machine */ SYSCTL_INT(_kern, OID_AUTO, dfbt_vhba_enable, CTLFLAG_RW, &vhba_enable, 0, "virtual HBA scripts DIAG dance on hard/soft reset writes"); /* virtual HBA observation of production register writes */ static u_int vhba_hard_resets, vhba_soft_resets, vhba_mbox_starts; static u_int vhba_last_ctl_write; SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_hard_resets, CTLFLAG_RD, &vhba_hard_resets, 0, "HARD_RESET(0x80) writes the driver made"); SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_soft_resets, CTLFLAG_RD, &vhba_soft_resets, 0, "SOFT_RESET(0x40) writes the driver made"); SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_mbox_starts, CTLFLAG_RD, &vhba_mbox_starts, 0, "BOP_START_MBOX writes the driver made"); SYSCTL_UINT(_kern, OID_AUTO, dfbt_vhba_last_ctl, CTLFLAG_RD, &vhba_last_ctl_write, 0, "last CONTROL_REG value the driver wrote"); typedef void (*btaction_fn)(struct cam_sim *, union ccb *); typedef void (*btdone_fn)(struct bt_softc *, struct bt_ccb *, bt_mbi_comp_code_t); typedef void (*bttimeout_fn)(void *); typedef int (*btreset_fn)(struct bt_softc *, int); typedef void (*btintr_fn)(struct bt_softc *); /* ------------------------------------------------------------------ */ /* Fabricated post-attach state */ /* ------------------------------------------------------------------ */ #define RIG_MAX_CCBS 16 #define RIG_UNIT 9 static struct bt_softc *bt; static struct bsd_device rig_dev; /* fabricated device_t */ static struct resource vport; /* fabricated port resource (MEM tag) */ static uint8_t *regpage; /* virtual HBA register file */ static struct cam_sim *rig_sim; static struct cam_path *io_path; /* target 3 lun 0 on our bus */ static struct cam_periph rig_periph; /* fabricated periph: type only */ static cdev_t rig_cdev; static struct thread *vhba_td; /* async event bookkeeping */ static u_int async_bdr_count, async_busreset_count; /* CCB registry */ struct rig_cctx { union ccb *ccb; struct bt_ccb *bccb; uint32_t bccb_phys; u_int comp_count; uint32_t last_status; }; static struct rig_cctx cctx[32]; static int ncctx; static d_open_t rig_open; static d_close_t rig_close; static d_ioctl_t rig_ioctl; static struct dev_ops rig_ops = { { "dfbt", 0, D_MPSAFE, NULL, 0, 0 }, .d_open = rig_open, .d_close = rig_close, .d_ioctl = rig_ioctl, }; /* ioctls */ struct rig_queue_args { uint32_t func_code; /* XPT_SCSI_IO | XPT_RESET_DEV */ uint32_t timeout_ms; uint32_t dxfer_len; /* 0 => CAM_DIR_NONE */ uint32_t dir_in; /* 1 => CAM_DIR_IN */ }; struct rig_calcgeom_args { uint32_t block_size; uint32_t volume_size; uint32_t heads, secs_per_track; uint32_t cylinders; uint32_t status; }; struct rig_comp_args { uint32_t use_recovery; /* target the recovery BCCB */ uint32_t ccb_id; uint8_t comp_code; uint8_t btstat; uint8_t sdstat; uint8_t _pad; }; struct rig_state { int32_t err; uint32_t ccb_id; uint64_t bccb_kva; uint64_t recovery_kva; uint32_t bccb_flags; uint32_t recovery_flags; uint32_t hccb_opcode, hccb_cmd_len, hccb_sense_len; uint32_t hccb_datain, hccb_dataout; uint32_t hccb_btstat, hccb_sdstat; uint32_t ccb_status; uint32_t timer_pending; uint32_t dmamap_nsegs; uint32_t on_pending_list; uint32_t comp_count; uint32_t last_status; uint32_t async_bdr, async_busreset; uint32_t vregs[4]; uint32_t outbox_ccbaddr, outbox_action; uint32_t active_ccbs, free_list_walk, pending_walk; }; struct rig_vreg_args { uint32_t off, val; }; #define RIGIOC_QUEUE _IOWR('B', 1, struct rig_queue_args) #define RIGIOC_SNAP _IOWR('B', 2, struct rig_state) #define RIGIOC_COMP _IOWR('B', 3, struct rig_comp_args) #define RIGIOC_TIMEOUT _IOWR('B', 4, struct rig_state) /* + ccb_id in st */ #define RIGIOC_CALCGEOM _IOWR('B', 5, struct rig_calcgeom_args) #define RIGIOC_BTRESET _IOWR('B', 6, struct rig_state) #define RIGIOC_VREG _IOWR('B', 7, struct rig_vreg_args) #define RIGIOC_HOLD _IOWR('B', 8, struct rig_state) #define RIGIOC_REL _IOWR('B', 9, struct rig_state) #define RIGIOC_RACE_FIRE _IOWR('B', 10, struct rig_state) #define RIGIOC_SIMQ_REL _IOWR('B', 11, struct rig_state) static void rig_done(struct cam_periph *periph, union ccb *done_ccb); static void rig_async(void *softc, u_int32_t code, struct cam_path *path, void *args); static __inline uint32_t rig_ccbvtop(struct bt_ccb *bccb) { return (bt->bt_ccb_physbase + (uint32_t)((caddr_t)bccb - (caddr_t)bt->bt_ccb_array)); } static __inline struct bt_ccb * rig_ccbptov(uint32_t ccb_addr) { return (bt->bt_ccb_array + ((struct bt_ccb *)(uintptr_t)ccb_addr - (struct bt_ccb *)(uintptr_t)bt->bt_ccb_physbase)); } static void rig_capture(struct rig_state *st, struct rig_cctx *cx) { struct ccb_hdr *ch; struct bt_ccb *b; int i; bzero(st, sizeof(*st)); st->err = 0; st->async_bdr = async_bdr_count; st->async_busreset = async_busreset_count; for (i = 0; i < 4; i++) st->vregs[i] = regpage[i]; st->outbox_ccbaddr = bt->cur_outbox->ccb_addr; st->outbox_action = bt->cur_outbox->action_code; st->recovery_kva = (uint64_t)(uintptr_t)bt->recovery_bccb; st->recovery_flags = bt->recovery_bccb ? bt->recovery_bccb->flags : ~0u; st->active_ccbs = bt->active_ccbs; st->free_list_walk = 0; SLIST_FOREACH(b, &bt->free_bt_ccbs, links) st->free_list_walk++; st->pending_walk = 0; LIST_FOREACH(ch, &bt->pending_ccbs, sim_links.le) st->pending_walk++; if (cx == NULL || cx->bccb == NULL) return; st->ccb_id = cx - cctx; st->bccb_kva = (uint64_t)(uintptr_t)cx->bccb; st->bccb_flags = cx->bccb->flags; st->hccb_opcode = cx->bccb->hccb.opcode; st->hccb_cmd_len = cx->bccb->hccb.cmd_len; st->hccb_sense_len = cx->bccb->hccb.sense_len; st->hccb_datain = cx->bccb->hccb.datain; st->hccb_dataout = cx->bccb->hccb.dataout; st->hccb_btstat = cx->bccb->hccb.btstat; st->hccb_sdstat = cx->bccb->hccb.sdstat; st->ccb_status = cx->ccb ? cx->ccb->ccb_h.status : ~0u; st->timer_pending = callout_pending(&cx->bccb->timer); st->dmamap_nsegs = cx->bccb->dmamap ? 1 : 0; /* map exists */ st->on_pending_list = 0; LIST_FOREACH(ch, &bt->pending_ccbs, sim_links.le) { if ((struct bt_ccb *)(uintptr_t)ch->spriv_ptr0 == cx->bccb) { st->on_pending_list = 1; break; } } st->comp_count = cx->comp_count; st->last_status = cx->last_status; } static void rig_done(struct cam_periph *periph, union ccb *done_ccb) { int i; for (i = 0; i < ncctx; i++) { if (cctx[i].ccb == done_ccb) { cctx[i].comp_count++; cctx[i].last_status = done_ccb->ccb_h.status; return; } } kprintf("[rig] rig_done: unknown ccb %p\n", done_ccb); } static void rig_async(void *softc, u_int32_t code, struct cam_path *path, void *args) { if (code == AC_SENT_BDR) async_bdr_count++; if (code == AC_BUS_RESET) async_busreset_count++; kprintf("[rig] async event 0x%x fired (bdr=%u busrst=%u)\n", code, async_bdr_count, async_busreset_count); } /* ------------------------------------------------------------------ */ /* busdma map callbacks (mirror btmapmboxes/btmapccbs) */ /* ------------------------------------------------------------------ */ static void rig_mapmboxes(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bt->mailbox_physbase = segs->ds_addr; } static void rig_mapccbs(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bt->bt_ccb_physbase = segs->ds_addr; } /* ------------------------------------------------------------------ */ /* Virtual HBA state machine: on a HARD/SOFT reset write to the */ /* CONTROL_REG, assert DIAG_ACTIVE briefly, then come ready -- so the */ /* production btreset() succeeds against scripted registers. */ /* Disable via kern.dfbt.vhba_enable=0 to keep DIAG_ACTIVE stuck */ /* (DF-4098 btreset panic variant). */ /* ------------------------------------------------------------------ */ static void rig_vhba_thread(void *arg) { uint8_t prev = 0; u_int prev_cmd = 0; while (vhba_td != NULL) { uint8_t ctl = regpage[CONTROL_REG]; uint8_t cmd = regpage[COMMAND_REG]; if (cmd != prev_cmd) { if (cmd == BOP_START_MBOX) vhba_mbox_starts++; prev_cmd = cmd; } if (vhba_enable && prev == 0 && (ctl & (HARD_RESET | SOFT_RESET)) != 0) { vhba_last_ctl_write = ctl; if (ctl & HARD_RESET) vhba_hard_resets++; if (ctl & SOFT_RESET) vhba_soft_resets++; /* reset request seen: run the "diagnostics" */ regpage[STATUS_REG] = DIAG_ACTIVE; tsleep(®page, 0, "vhba", hz / 8); regpage[STATUS_REG] = HA_READY | INIT_REQUIRED; regpage[CONTROL_REG] = 0; } prev = ctl & (HARD_RESET | SOFT_RESET); tsleep(&vhba_td, 0, "vhbaidle", hz / 20); } kthread_exit(); } /* ------------------------------------------------------------------ */ /* Setup: fabricate the post-attach softc and register the real SIM */ /* ------------------------------------------------------------------ */ static int rig_setup(void) { struct cam_devq *devq; u_int boxes; int error; if (bt != NULL) return (0); regpage = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT | M_ZERO); bzero(&vport, sizeof(vport)); vport.r_bustag = 1; /* X86_64_BUS_SPACE_MEM */ vport.r_bushandle = (bus_space_handle_t)(uintptr_t)regpage; regpage[STATUS_REG] = HA_READY; bzero(&rig_dev, sizeof(rig_dev)); rig_dev.nameunit = "bt9"; bt = kmalloc(sizeof(*bt), M_DEVBUF, M_INTWAIT | M_ZERO); SLIST_INIT(&bt->free_bt_ccbs); LIST_INIT(&bt->pending_ccbs); SLIST_INIT(&bt->sg_maps); bt->dev = &rig_dev; bt->port = &vport; lockinit(&bt->lock, "bt", 0, LK_CANRECURSE); /* mirrors bt_init_softc */ bt->max_ccbs = RIG_MAX_CCBS; boxes = RIG_MAX_CCBS; bt->num_boxes = boxes; bt->scsi_id = 7; ksnprintf(bt->firmware_ver, sizeof(bt->firmware_ver), "5.07"); ksnprintf(bt->model, sizeof(bt->model), "958"); bt->extended_trans = 1; bt->strict_rr = 1; bt->init_level = 4; /* mailboxes not re-inited by btreset */ /* real busdma tags/maps exactly as bt_init() builds them */ error = bus_dma_tag_create(/*parent*/ NULL, /*alignment*/ 1, /*boundary*/ 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, MAXBSIZE, BT_NSEG, BUS_SPACE_MAXSIZE_32BIT, BUS_DMA_ALLOCNOW, &bt->buffer_dmat); if (error) { kprintf("[rig] buffer_dmat tag fail %d\n", error); return (error); } error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, boxes * (sizeof(bt_mbox_in_t) + sizeof(bt_mbox_out_t)), 1, BUS_SPACE_MAXSIZE_32BIT, 0, &bt->mailbox_dmat); if (error) { kprintf("[rig] mailbox_dmat tag fail\n"); return (error); } error = bus_dmamem_alloc(bt->mailbox_dmat, (void **)&bt->out_boxes, BUS_DMA_NOWAIT, &bt->mailbox_dmamap); if (error) { kprintf("[rig] mailbox alloc fail\n"); return (error); } bus_dmamap_load(bt->mailbox_dmat, bt->mailbox_dmamap, bt->out_boxes, boxes * (sizeof(bt_mbox_in_t) + sizeof(bt_mbox_out_t)), rig_mapmboxes, NULL, 0); bt->in_boxes = (bt_mbox_in_t *)&bt->out_boxes[boxes]; error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, RIG_MAX_CCBS * sizeof(struct bt_ccb), 1, BUS_SPACE_MAXSIZE_32BIT, 0, &bt->ccb_dmat); if (error) { kprintf("[rig] ccb_dmat tag fail\n"); return (error); } error = bus_dmamem_alloc(bt->ccb_dmat, (void **)&bt->bt_ccb_array, BUS_DMA_NOWAIT, &bt->ccb_dmamap); if (error) { kprintf("[rig] ccb array alloc fail\n"); return (error); } bus_dmamap_load(bt->ccb_dmat, bt->ccb_dmamap, bt->bt_ccb_array, RIG_MAX_CCBS * sizeof(struct bt_ccb), rig_mapccbs, NULL, 0); bzero(bt->bt_ccb_array, RIG_MAX_CCBS * sizeof(struct bt_ccb)); error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, PAGE_SIZE, 1, BUS_SPACE_MAXSIZE_32BIT, 0, &bt->sg_dmat); if (error) { kprintf("[rig] sg_dmat tag fail\n"); return (error); } /* mailbox pointers exactly as btinitmboxes() leaves them */ bzero(bt->in_boxes, sizeof(bt_mbox_in_t) * boxes); bzero(bt->out_boxes, sizeof(bt_mbox_out_t) * boxes); bt->cur_inbox = bt->in_boxes; bt->last_inbox = bt->in_boxes + boxes - 1; bt->cur_outbox = bt->out_boxes; bt->last_outbox = bt->out_boxes + boxes - 1; /* register the REAL sim: production btaction/btpoll over bt */ devq = cam_simq_alloc(RIG_MAX_CCBS); if (devq == NULL) return (ENOMEM); rig_sim = cam_sim_alloc((sim_action_func)a_btaction, (sim_poll_func)a_btpoll, "bt", bt, RIG_UNIT, &bt->lock, RIG_MAX_CCBS, RIG_MAX_CCBS, devq); cam_simq_release(devq); if (rig_sim == NULL) return (ENOMEM); /* bt_attach holds bt->lock across registration (bt.c:769-784) */ lockmgr(&bt->lock, LK_EXCLUSIVE); bt->sim = rig_sim; /* bt_attach:922 keeps its sim in the softc */ if (xpt_bus_register(rig_sim, 0) != CAM_SUCCESS) { lockmgr(&bt->lock, LK_RELEASE); cam_sim_free(rig_sim); rig_sim = NULL; return (ENXIO); } if (xpt_create_path(&bt->path, /*periph*/ NULL, cam_sim_path(rig_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { lockmgr(&bt->lock, LK_RELEASE); return (ENXIO); } if (xpt_create_path(&io_path, /*periph*/ NULL, cam_sim_path(rig_sim), 3, 0) != CAM_REQ_CMP) { lockmgr(&bt->lock, LK_RELEASE); return (ENXIO); } /* fabricated periph: xpt_done reads path->periph->type only * (cam_xpt.c:4903), must be CAM_PERIPH_BIO. struct cam_path is * private to cam_xpt.c; periph is its first field (cam_xpt.c:202), * so write it via offset 0. */ bzero(&rig_periph, sizeof(rig_periph)); rig_periph.type = CAM_PERIPH_BIO; rig_periph.path = io_path; rig_periph.periph_name = "dfbt"; *(struct cam_periph **)(void *)io_path = &rig_periph; xpt_register_async(AC_SENT_BDR | AC_BUS_RESET, rig_async, NULL, io_path); lockmgr(&bt->lock, LK_RELEASE); error = kthread_create(rig_vhba_thread, NULL, &vhba_td, "dfbt_vhba"); if (error) vhba_td = NULL; /* prime magic prologue words for host-side verification */ m_btaction = *(volatile uint64_t *)a_btaction; m_btpoll = *(volatile uint64_t *)a_btpoll; m_btdone = *(volatile uint64_t *)a_btdone; m_bttimeout= *(volatile uint64_t *)a_bttimeout; m_btreset = *(volatile uint64_t *)a_btreset; m_btintr = *(volatile uint64_t *)a_bt_intr_locked; kprintf("[rig] dfbt rig up: softc=%p sim=%p path_id=%d " "ccb_physbase=0x%08x mbox_physbase=0x%08x\n", bt, rig_sim, cam_sim_path(rig_sim), bt->bt_ccb_physbase, (unsigned)bt->mailbox_physbase); return (0); } /* ------------------------------------------------------------------ */ /* stale-handler wrapper for DF-4101: production bttimeout on a thread */ /* ------------------------------------------------------------------ */ static struct bt_ccb *race_bccb; static volatile int race_entered, race_done; static void rig_race_thread(void *arg) { kprintf("[rig] stale handler thread: calling production bttimeout(" "%p), will block on bt->lock\n", race_bccb); race_entered = 1; ((bttimeout_fn)a_bttimeout)(race_bccb); /* blocks on bt->lock */ race_done = 1; kprintf("[rig] stale handler thread: bttimeout returned\n"); kthread_exit(); } /* ------------------------------------------------------------------ */ /* ioctl dispatch */ /* ------------------------------------------------------------------ */ static int rig_open(struct dev_open_args *ap) { return (0); } static int rig_close(struct dev_close_args *ap) { return (0); } static int rig_ioctl(struct dev_ioctl_args *ap) { u_long cmd = ap->a_cmd; caddr_t data = ap->a_data; struct rig_state *st; int error = 0; if (bt == NULL) { if (a_btaction == 0 || a_btdone == 0 || a_bttimeout == 0 || a_btreset == 0 || a_bt_intr_locked == 0 || a_btpoll == 0) { kprintf("[rig] addresses not primed\n"); return (ENXIO); } error = rig_setup(); if (error) return (error); } switch (cmd) { case RIGIOC_QUEUE: { struct rig_queue_args *qa = (struct rig_queue_args *)data; union ccb *ccb; struct rig_cctx *cx; if (ncctx >= 30) return (ENOMEM); ccb = xpt_alloc_ccb(); xpt_setup_ccb(&ccb->ccb_h, io_path, /*priority*/ 1); ccb->ccb_h.func_code = qa->func_code; ccb->ccb_h.cbfcnp = rig_done; ccb->ccb_h.timeout = qa->timeout_ms; if (qa->func_code == XPT_SCSI_IO) { ccb->csio.cdb_len = 6; ccb->csio.cdb_io.cdb_bytes[0] = 0x00; /* TUR */ ccb->csio.sense_len = 0; if (qa->dir_in && qa->dxfer_len) { ccb->ccb_h.flags = CAM_DIR_IN; ccb->csio.dxfer_len = qa->dxfer_len; ccb->csio.data_ptr = kmalloc(qa->dxfer_len, M_DEVBUF, M_INTWAIT | M_ZERO); } else { ccb->ccb_h.flags = CAM_DIR_NONE; ccb->csio.dxfer_len = 0; ccb->csio.data_ptr = NULL; } } else { ccb->ccb_h.flags = CAM_DIR_NONE; } cx = &cctx[ncctx]; cx->ccb = ccb; cx->bccb = NULL; cx->bccb_phys = 0; cx->comp_count = 0; cx->last_status = 0; ncctx++; kprintf("[rig] QUEUE: xpt_action(func=%x) id=%d\n", qa->func_code, ncctx - 1); lockmgr(&bt->lock, LK_EXCLUSIVE); xpt_action(ccb); /* production: XPT -> btaction */ kprintf("[rig] QUEUE post: pinfo.index=%d devq.entries=%u " "send_openings=%u send_active=%u sim_openings=%d\n", ccb->ccb_h.pinfo.index, rig_sim->devq->send_queue.entries, rig_sim->devq->send_openings, rig_sim->devq->send_active, rig_sim->max_dev_openings); lockmgr(&bt->lock, LK_RELEASE); /* btaction stored the bccb pointer for us */ cx->bccb = cx->bccb ? cx->bccb : (struct bt_ccb *)(uintptr_t)ccb->ccb_h.spriv_ptr0; if (cx->bccb) cx->bccb_phys = rig_ccbvtop(cx->bccb); st = (struct rig_state *)data; rig_capture(st, cx); st->ccb_id = ncctx - 1; kprintf("[rig] QUEUE done: id=%d bccb=%p opcode=0x%02x " "cmd_len=%u sense_len=%u bccb_phys=0x%08x\n", ncctx - 1, cx->bccb, st->hccb_opcode, st->hccb_cmd_len, st->hccb_sense_len, cx->bccb_phys); break; } case RIGIOC_SNAP: { st = (struct rig_state *)data; lockmgr(&bt->lock, LK_EXCLUSIVE); rig_capture(st, st->ccb_id < ncctx ? &cctx[st->ccb_id] : NULL); lockmgr(&bt->lock, LK_RELEASE); break; } case RIGIOC_COMP: { struct rig_comp_args *ca = (struct rig_comp_args *)data; struct bt_ccb *bccb; struct rig_cctx *cx = NULL; if (ca->use_recovery) { bccb = bt->recovery_bccb; } else { if (ca->ccb_id >= ncctx) return (EINVAL); cx = &cctx[ca->ccb_id]; bccb = cx->bccb; } if (bccb == NULL) { kprintf("[rig] COMP: target bccb is NULL (command was " "never queued -- REQUEUE'd?); refusing\n"); return (EINVAL); } kprintf("[rig] COMP: firmware writes completion: bccb=%p " "comp_code=0x%02x btstat=0x%02x sdstat=0x%02x -> " "production btdone()\n", bccb, ca->comp_code, ca->btstat, ca->sdstat); /* the host adapter writes status into CCB DMA memory */ if (ca->btstat || ca->sdstat) { bccb->hccb.btstat = ca->btstat; bccb->hccb.sdstat = ca->sdstat; } lockmgr(&bt->lock, LK_EXCLUSIVE); ((btdone_fn)a_btdone)(bt, bccb, ca->comp_code); lockmgr(&bt->lock, LK_RELEASE); st = (struct rig_state *)data; bzero(st, sizeof(*st)); if (cx) { rig_capture(st, cx); } else { rig_capture(st, NULL); st->bccb_kva = (uint64_t)(uintptr_t)bccb; st->bccb_flags = bccb->flags; st->hccb_opcode = bccb->hccb.opcode; st->hccb_btstat = bccb->hccb.btstat; } kprintf("[rig] COMP done: flags=0x%x pending_list=%u " "timer=%u nsegs=%u ccb_status=0x%x\n", st->bccb_flags, st->on_pending_list, st->timer_pending, st->dmamap_nsegs, st->ccb_status); break; } case RIGIOC_TIMEOUT: { st = (struct rig_state *)data; if (st->ccb_id >= (uint32_t)ncctx || cctx[st->ccb_id].bccb == NULL) return (EINVAL); kprintf("[rig] TIMEOUT: production bttimeout(%p)\n", cctx[st->ccb_id].bccb); ((bttimeout_fn)a_bttimeout)(cctx[st->ccb_id].bccb); lockmgr(&bt->lock, LK_EXCLUSIVE); rig_capture(st, &cctx[st->ccb_id]); lockmgr(&bt->lock, LK_RELEASE); kprintf("[rig] TIMEOUT done: flags=0x%x recovery_flags=0x%x " "ctlreg=0x%02x outbox_ccb=0x%08x outbox_act=0x%02x\n", st->bccb_flags, st->recovery_flags, st->vregs[0], st->outbox_ccbaddr, st->outbox_action); break; } case RIGIOC_RACE_FIRE: { st = (struct rig_state *)data; if (st->ccb_id >= (uint32_t)ncctx || cctx[st->ccb_id].bccb == NULL) return (EINVAL); race_bccb = cctx[st->ccb_id].bccb; race_entered = race_done = 0; error = kthread_create(rig_race_thread, NULL, NULL, "dfbt_stale"); if (error) return (error); /* wait for the thread to *enter* bttimeout (before lock) */ int n; for (n = 0; n < 100 && !race_entered; n++) tsleep(&race_entered, 0, "rigent", hz / 20); kprintf("[rig] RACE_FIRE: stale bttimeout entered=%d " "(it is now blocked acquiring bt->lock)\n", race_entered); break; } case RIGIOC_CALCGEOM: { struct rig_calcgeom_args *ga = (struct rig_calcgeom_args *)data; union ccb *ccb = xpt_alloc_ccb(); xpt_setup_ccb(&ccb->ccb_h, io_path, /*priority*/ 1); ccb->ccb_h.func_code = XPT_CALC_GEOMETRY; ccb->ccb_h.cbfcnp = rig_done; ccb->ccg.block_size = ga->block_size; ccb->ccg.volume_size = ga->volume_size; ccb->ccg.heads = 0; ccb->ccg.secs_per_track = 0; ccb->ccg.cylinders = 0; kprintf("[rig] CALCGEOM: xpt_action(block_size=%u " "volume_size=%u)\n", ga->block_size, ga->volume_size); lockmgr(&bt->lock, LK_EXCLUSIVE); xpt_action(ccb); /* production XPT -> btaction */ lockmgr(&bt->lock, LK_RELEASE); /* if we get here the geometry completed without a trap */ ga->heads = ccb->ccg.heads; ga->secs_per_track = ccb->ccg.secs_per_track; ga->cylinders = ccb->ccg.cylinders; ga->status = ccb->ccb_h.status; kprintf("[rig] CALCGEOM survived: heads=%u spt=%u cyl=%u " "status=0x%x\n", ga->heads, ga->secs_per_track, ga->cylinders, ga->status); break; } case RIGIOC_BTRESET: { st = (struct rig_state *)data; kprintf("[rig] BTRESET: production btreset(hard=%d)\n", st->hccb_opcode); lockmgr(&bt->lock, LK_EXCLUSIVE); st->err = ((btreset_fn)a_btreset)(bt, st->hccb_opcode ? 1 : 0); lockmgr(&bt->lock, LK_RELEASE); rig_capture(st, NULL); kprintf("[rig] BTRESET done: err=%d status_reg=0x%02x\n", st->err, st->vregs[0]); break; } case RIGIOC_VREG: { struct rig_vreg_args *va = (struct rig_vreg_args *)data; if (va->off > 3) return (EINVAL); regpage[va->off] = va->val; break; } case RIGIOC_HOLD: lockmgr(&bt->lock, LK_EXCLUSIVE); kprintf("[rig] HOLD: bt->lock held by ioctl thread\n"); break; case RIGIOC_REL: lockmgr(&bt->lock, LK_RELEASE); kprintf("[rig] REL: bt->lock released\n"); break; case RIGIOC_SIMQ_REL: lockmgr(&bt->lock, LK_EXCLUSIVE); xpt_release_simq(rig_sim, /*run_queue*/ TRUE); lockmgr(&bt->lock, LK_RELEASE); kprintf("[rig] SIMQ released\n"); break; default: return (ENOTTY); } return (0); } /* ------------------------------------------------------------------ */ /* module boilerplate */ /* ------------------------------------------------------------------ */ static int dfbt_modevent(module_t mod, int type, void *data) { switch (type) { case MOD_LOAD: rig_cdev = make_dev(&rig_ops, 0, UID_ROOT, GID_WHEEL, 0600, "dfbt%d", 0); kprintf("[rig] dfbt rig loaded (prime kern.dfbt.a_* then " "open /dev/dfbt0)\n"); return (0); case MOD_UNLOAD: if (bt != NULL) { kprintf("[rig] refusing unload while rig is set up " "(reboot instead)\n"); return (EBUSY); } if (rig_cdev != NULL) destroy_dev(rig_cdev); return (0); default: return (EOPNOTSUPP); } } DEV_MODULE(dfbt_rig, dfbt_modevent, NULL);