/* * i915_rcu_harness.c - replica of the i915 request-object lockless-RCU * lifetime protocol as it behaves under DragonFly's Linux-compat shims * (DF-4329). * * Upstream protocol (Linux v4.19, drivers/gpu/drm/i915/i915_request.c/h): * - request objects live in a dedicated SLAB_TYPESAFE_BY_RCU slab: * while a reader holds rcu_read_lock(), the slab *pages* cannot be * freed, and freed objects can only be reused as other requests; * - __i915_gem_active_get_rcu() (i915_request.h:514) chases * request->global_seqno and request->engine locklessly and takes a * reference with dma_fence_get_rcu() == kref_get_unless_zero(): * it refuses to resurrect a refcount==0 (freed) fence; * - i915_request_alloc() relies on this ("Beware: Dragons be flying * overhead", i915_request.c:703-731) and its OOM path waits for * readers with cond_synchronize_rcu() before reclaiming memory * (i915_request.c:738-741, backported from upstream). * * DragonFly port reality: * - sys/dev/drm/include/linux/rcupdate.h:94-96: * synchronize_rcu() -> * cond_synchronize_rcu(x) -> cpu_mfence() * get_state_synchronize_rcu() -> 0 * and rcu_read_lock() == preempt_disable(). There is NO grace period. * - sys/dev/drm/include/linux/slab.h: SLAB_TYPESAFE_BY_RCU is defined * but ignored; kmem_cache_alloc() == kmalloc(size, M_DRM) from the * generic per-type zone shared by EVERY drm allocation; * kmem_cache_free() == kfree() -> freed request memory is immediately * reusable by any other same-size M_DRM allocation. * - sys/dev/drm/include/linux/dma-fence.h:98: dma_fence_get_rcu() == * plain kref_get() -> UNCONDITIONAL increment, no unless-zero check. * * Net effect, replicated below in a sequenced (race-free-to-arrange) form; * every real interleaving that hits the same window is exactly this state: * * 1. request r allocated from the generic zone, published in an active * tracker (i915_gem_active.request). * 2. r retires -> i915_fence_release -> kmem_cache_free -> kfree(r). * 3. an unrelated M_DRM-style allocation instantly reuses the chunk. * 4. a lockless reader holding rcu_read_lock() (== crit section) still * sees r in the tracker and chases r->engine - which now contains * the reuser's data - and dereferences it (upstream: * __i915_request_completed -> intel_engine_has_completed(rq->engine, * seqno), i915_request.h:318-322). * * sysctl kern.i915rcu_run: * 1 = log-only proof: instant reuse + refcount resurrection + would-be * engine pointer (no fault) * 2 = continue the reader protocol and actually dereference the * attacker-influenced engine pointer -> EXPECT KERNEL PANIC on the * marker value. */ #include #include #include #include #include #include #include #include #include #include #include #define REQ_SZ 384 /* i915_request size class */ #define ENGINE_MARK 0x0000515151515151UL /* "attacker" engine ptr */ #define CHASE_OFF 0x18 /* inside engine->timeline */ /* mirrors the first fields of struct i915_request (i915_request.h:89-99): * struct dma_fence fence; // fence.refcount is at offset 0 (dma-fence.h) * spinlock_t lock; * struct drm_i915_private *i915; * struct i915_gem_context *gem_context; * struct intel_engine_cs *engine; <- the pointer readers chase * u32 global_seqno; */ struct fake_req { volatile int refcount; /* == &rq->fence.refcount @ 0 */ void *lock; void *i915; void *gem_context; void *engine; /* the chased pointer */ volatile unsigned global_seqno; char pad[REQ_SZ - 40]; /* total == 384-byte size class */ }; MALLOC_DEFINE(M_I915H, "i915rcu", "i915 rcu harness"); static struct fake_req * volatile active_req; /* i915_gem_active.request */ static int run_replica(int mode) { struct fake_req *r, *unrelated, *p; int resurrected = -1; void *engine; unsigned seqno; /* 1. request allocated from the generic zone and published */ r = kmalloc(sizeof(*r), M_I915H, M_WAITOK | M_ZERO); r->refcount = 1; r->engine = (void *)(uintptr_t)0x0000ffff00000000UL; /* real-ish */ r->global_seqno = 1; active_req = r; /* rcu_assign_pointer == store */ kprintf("i915rcu: [1] request r=%p published (refcount=1, " "engine=%p)\n", r, r->engine); /* 2. retire -> i915_fence_release -> kmem_cache_free -> kfree */ kprintf("i915rcu: [2] retire: refcount 1->0, i915_fence_release:" " kmem_cache_free(i915->requests, rq) -> kfree(%p)\n", r); kfree(r, M_I915H); /* upstream would now have SLAB_TYPESAFE_BY_RCU keep the chunk a * request; DFly: the chunk is on the generic zone freelist. */ /* 3. unrelated same-size allocation instantly reuses the chunk */ unrelated = kmalloc(sizeof(*unrelated), M_I915H, M_WAITOK | M_ZERO); kprintf("i915rcu: [3] unrelated kmalloc(%zu) -> %p %s r\n", sizeof(*unrelated), unrelated, (void *)unrelated == (void *)r ? "== REUSED (no grace period," " SLAB_TYPESAFE_BY_RCU unenforced)" : "!= r (zone reused page?)"); if ((void *)unrelated != (void *)r) { kprintf("i915rcu: NOTE: allocator handed a different chunk; " "retrying publish with the new chunk to keep the replica " "sequential\n"); kfree(unrelated, M_I915H); r = kmalloc(sizeof(*r), M_I915H, M_WAITOK | M_ZERO); r->refcount = 1; active_req = r; kfree(r, M_I915H); unrelated = kmalloc(sizeof(*unrelated), M_I915H, M_WAITOK | M_ZERO); kprintf("i915rcu: [3b] retry: r=%p unrelated=%p eq=%d\n", r, unrelated, (void *)unrelated == (void *)r); } /* the "attacker" fills the reused object (any drm struct with * user-controlled fields would do); we plant the engine-pointer * marker at the exact offset readers chase */ unrelated->engine = (void *)ENGINE_MARK; unrelated->global_seqno = 0x42; kprintf("i915rcu: [4] reused object poisoned: engine@%p = " "0x%016jx (attacker-influenced), global_seqno=0x%x\n", &unrelated->engine, (uintmax_t)ENGINE_MARK, 0x42); /* refcount-resurrection sub-proof (dma_fence_get_rcu shim): * do it on a scratch copy so the poisoned object stays pristine */ { struct fake_req scratch; scratch.refcount = 0; /* freed fence */ /* shim: kref_get == refcount_inc (unconditional) */ atomic_add_int(&scratch.refcount, 1); resurrected = scratch.refcount; /* 1: resurrected */ kprintf("i915rcu: [5] dma_fence_get_rcu() shim replica: " "refcount 0 -> %d (upstream kref_get_unless_zero would " "REFUSE and return NULL)\n", resurrected); } /* 6. the lockless reader, exactly __i915_gem_active_get_rcu() under * rcu_read_lock() == preempt_disable() == crit_enter() */ kprintf("i915rcu: [6] reader: rcu_read_lock()=crit_enter(); " "p = rcu_dereference(active.request)...\n"); crit_enter(); /* the port's rcu_read_lock */ p = active_req; /* rcu_dereference */ kprintf("i915rcu: [6] reader sees p=%p (stale, freed & reused)\n", p); /* i915_request_completed(p) (i915_request.h:325) first reads * global_seqno, then, if nonzero, chases p->engine: */ seqno = p->global_seqno; kprintf("i915rcu: [6] p->global_seqno=0x%x\n", seqno); if (seqno) { engine = p->engine; /* == ENGINE_MARK */ kprintf("i915rcu: [6] chasing p->engine = 0x%016jx " "(intel_engine_has_completed(rq->engine, seqno))\n", (uintmax_t)(uintptr_t)engine); if (mode == 2) { kprintf("i915rcu: [7] mode 2: READING *(u32 *)" "(engine+0x%x) -> EXPECT PANIC ON 0x%016jx\n", CHASE_OFF, (uintmax_t)(uintptr_t)engine + CHASE_OFF); kprintf("i915rcu: read value would be 0x%08x\n", *(volatile unsigned *)((char *)engine + CHASE_OFF)); kprintf("i915rcu: survived chase (unexpected)\n"); } else { kprintf("i915rcu: [7] mode 1: would read *(u32 *)" "(engine+0x%x) at 0x%016jx (SKIPPED - no fault in " "log mode)\n", CHASE_OFF, (uintmax_t)(uintptr_t)engine + CHASE_OFF); } } crit_exit(); /* rcu_read_unlock */ kfree(unrelated, M_I915H); active_req = NULL; return (0); } static int run_sysctl(SYSCTL_HANDLER_ARGS) { int error, v = 0; error = sysctl_handle_int(oidp, &v, 0, req); if (error || req->newptr == NULL) return (error); if (v == 1 || v == 2) run_replica(v); return (0); } static SYSCTL_NODE(_kern, OID_AUTO, i915rcu, CTLFLAG_RW, 0, "i915 rcu replica"); SYSCTL_PROC(_kern_i915rcu, OID_AUTO, run, CTLTYPE_INT | CTLFLAG_RW, 0, 0, run_sysctl, "I", "1=log-only 2=chase(panic)"); static int i915rcu_modevent(module_t mod, int type, void *data) { int error = 0; switch (type) { case MOD_LOAD: kprintf("i915rcu: loaded (sysctl kern.i915rcu.run=N)\n"); break; case MOD_UNLOAD: break; default: break; } return (error); } static struct moduledata i915_rcu_harness_mod = { "i915_rcu_harness", i915rcu_modevent, NULL }; DECLARE_MODULE(i915_rcu_harness, i915_rcu_harness_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); MODULE_VERSION(i915_rcu_harness, 1);