/*
 * i915_rcu_harness.c - replica of the i915 request-object lockless-RCU
 * lifetime protocol as it behaves under DragonFly's Linux-compat shims
 * (DF-4329).
 *
 * Upstream protocol (Linux v4.19, drivers/gpu/drm/i915/i915_request.c/h):
 *   - request objects live in a dedicated SLAB_TYPESAFE_BY_RCU slab:
 *     while a reader holds rcu_read_lock(), the slab *pages* cannot be
 *     freed, and freed objects can only be reused as other requests;
 *   - __i915_gem_active_get_rcu() (i915_request.h:514) chases
 *     request->global_seqno and request->engine locklessly and takes a
 *     reference with dma_fence_get_rcu() == kref_get_unless_zero():
 *     it refuses to resurrect a refcount==0 (freed) fence;
 *   - i915_request_alloc() relies on this ("Beware: Dragons be flying
 *     overhead", i915_request.c:703-731) and its OOM path waits for
 *     readers with cond_synchronize_rcu() before reclaiming memory
 *     (i915_request.c:738-741, backported from upstream).
 *
 * DragonFly port reality:
 *   - sys/dev/drm/include/linux/rcupdate.h:94-96:
 *         synchronize_rcu()            -> <empty statement>
 *         cond_synchronize_rcu(x)      -> cpu_mfence()
 *         get_state_synchronize_rcu()  -> 0
 *     and rcu_read_lock() == preempt_disable().  There is NO grace period.
 *   - sys/dev/drm/include/linux/slab.h: SLAB_TYPESAFE_BY_RCU is defined
 *     but ignored; kmem_cache_alloc() == kmalloc(size, M_DRM) from the
 *     generic per-type zone shared by EVERY drm allocation;
 *     kmem_cache_free() == kfree() -> freed request memory is immediately
 *     reusable by any other same-size M_DRM allocation.
 *   - sys/dev/drm/include/linux/dma-fence.h:98: dma_fence_get_rcu() ==
 *     plain kref_get() -> UNCONDITIONAL increment, no unless-zero check.
 *
 * Net effect, replicated below in a sequenced (race-free-to-arrange) form;
 * every real interleaving that hits the same window is exactly this state:
 *
 *   1. request r allocated from the generic zone, published in an active
 *      tracker (i915_gem_active.request).
 *   2. r retires -> i915_fence_release -> kmem_cache_free -> kfree(r).
 *   3. an unrelated M_DRM-style allocation instantly reuses the chunk.
 *   4. a lockless reader holding rcu_read_lock() (== crit section) still
 *      sees r in the tracker and chases r->engine - which now contains
 *      the reuser's data - and dereferences it (upstream:
 *      __i915_request_completed -> intel_engine_has_completed(rq->engine,
 *      seqno), i915_request.h:318-322).
 *
 * sysctl kern.i915rcu_run:
 *   1 = log-only proof: instant reuse + refcount resurrection + would-be
 *       engine pointer (no fault)
 *   2 = continue the reader protocol and actually dereference the
 *       attacker-influenced engine pointer -> EXPECT KERNEL PANIC on the
 *       marker value.
 */
#include <sys/param.h>
#include <sys/types.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/thread.h>
#include <sys/spinlock2.h>
#include <machine/cpufunc.h>
#include <machine/atomic.h>

#define REQ_SZ		384			/* i915_request size class */
#define ENGINE_MARK	0x0000515151515151UL	/* "attacker" engine ptr */
#define CHASE_OFF	0x18			/* inside engine->timeline */

/* mirrors the first fields of struct i915_request (i915_request.h:89-99):
 * struct dma_fence fence;   // fence.refcount is at offset 0 (dma-fence.h)
 * spinlock_t lock;
 * struct drm_i915_private *i915;
 * struct i915_gem_context *gem_context;
 * struct intel_engine_cs *engine;   <- the pointer readers chase
 * u32 global_seqno;
 */
struct fake_req {
	volatile int refcount;		/* == &rq->fence.refcount @ 0 */
	void *lock;
	void *i915;
	void *gem_context;
	void *engine;			/* the chased pointer */
	volatile unsigned global_seqno;
	char pad[REQ_SZ - 40];		/* total == 384-byte size class */
};

MALLOC_DEFINE(M_I915H, "i915rcu", "i915 rcu harness");

static struct fake_req * volatile active_req;	/* i915_gem_active.request */

static int
run_replica(int mode)
{
	struct fake_req *r, *unrelated, *p;
	int resurrected = -1;
	void *engine;
	unsigned seqno;

	/* 1. request allocated from the generic zone and published */
	r = kmalloc(sizeof(*r), M_I915H, M_WAITOK | M_ZERO);
	r->refcount = 1;
	r->engine = (void *)(uintptr_t)0x0000ffff00000000UL; /* real-ish */
	r->global_seqno = 1;
	active_req = r;			/* rcu_assign_pointer == store */
	kprintf("i915rcu: [1] request r=%p published (refcount=1, "
	    "engine=%p)\n", r, r->engine);

	/* 2. retire -> i915_fence_release -> kmem_cache_free -> kfree */
	kprintf("i915rcu: [2] retire: refcount 1->0, i915_fence_release:"
	    " kmem_cache_free(i915->requests, rq) -> kfree(%p)\n", r);
	kfree(r, M_I915H);

	/* upstream would now have SLAB_TYPESAFE_BY_RCU keep the chunk a
	 * request; DFly: the chunk is on the generic zone freelist. */

	/* 3. unrelated same-size allocation instantly reuses the chunk */
	unrelated = kmalloc(sizeof(*unrelated), M_I915H, M_WAITOK | M_ZERO);
	kprintf("i915rcu: [3] unrelated kmalloc(%zu) -> %p %s r\n",
	    sizeof(*unrelated), unrelated,
	    (void *)unrelated == (void *)r ? "== REUSED (no grace period,"
	    " SLAB_TYPESAFE_BY_RCU unenforced)" : "!= r (zone reused page?)");

	if ((void *)unrelated != (void *)r) {
		kprintf("i915rcu: NOTE: allocator handed a different chunk; "
		    "retrying publish with the new chunk to keep the replica "
		    "sequential\n");
		kfree(unrelated, M_I915H);
		r = kmalloc(sizeof(*r), M_I915H, M_WAITOK | M_ZERO);
		r->refcount = 1;
		active_req = r;
		kfree(r, M_I915H);
		unrelated = kmalloc(sizeof(*unrelated), M_I915H,
		    M_WAITOK | M_ZERO);
		kprintf("i915rcu: [3b] retry: r=%p unrelated=%p eq=%d\n",
		    r, unrelated, (void *)unrelated == (void *)r);
	}

	/* the "attacker" fills the reused object (any drm struct with
	 * user-controlled fields would do); we plant the engine-pointer
	 * marker at the exact offset readers chase */
	unrelated->engine = (void *)ENGINE_MARK;
	unrelated->global_seqno = 0x42;
	kprintf("i915rcu: [4] reused object poisoned: engine@%p = "
	    "0x%016jx (attacker-influenced), global_seqno=0x%x\n",
	    &unrelated->engine, (uintmax_t)ENGINE_MARK, 0x42);

	/* refcount-resurrection sub-proof (dma_fence_get_rcu shim):
	 * do it on a scratch copy so the poisoned object stays pristine */
	{
		struct fake_req scratch;
		scratch.refcount = 0;		/* freed fence */
		/* shim: kref_get == refcount_inc (unconditional) */
		atomic_add_int(&scratch.refcount, 1);
		resurrected = scratch.refcount;	/* 1: resurrected */
		kprintf("i915rcu: [5] dma_fence_get_rcu() shim replica: "
		    "refcount 0 -> %d (upstream kref_get_unless_zero would "
		    "REFUSE and return NULL)\n", resurrected);
	}

	/* 6. the lockless reader, exactly __i915_gem_active_get_rcu() under
	 *    rcu_read_lock() == preempt_disable() == crit_enter() */
	kprintf("i915rcu: [6] reader: rcu_read_lock()=crit_enter(); "
	    "p = rcu_dereference(active.request)...\n");
	crit_enter();				/* the port's rcu_read_lock */
	p = active_req;				/* rcu_dereference */
	kprintf("i915rcu: [6] reader sees p=%p (stale, freed & reused)\n", p);

	/* i915_request_completed(p) (i915_request.h:325) first reads
	 * global_seqno, then, if nonzero, chases p->engine: */
	seqno = p->global_seqno;
	kprintf("i915rcu: [6] p->global_seqno=0x%x\n", seqno);
	if (seqno) {
		engine = p->engine;		/* == ENGINE_MARK */
		kprintf("i915rcu: [6] chasing p->engine = 0x%016jx "
		    "(intel_engine_has_completed(rq->engine, seqno))\n",
		    (uintmax_t)(uintptr_t)engine);
		if (mode == 2) {
			kprintf("i915rcu: [7] mode 2: READING *(u32 *)"
			    "(engine+0x%x) -> EXPECT PANIC ON 0x%016jx\n",
			    CHASE_OFF,
			    (uintmax_t)(uintptr_t)engine + CHASE_OFF);
			kprintf("i915rcu: read value would be 0x%08x\n",
			    *(volatile unsigned *)((char *)engine + CHASE_OFF));
			kprintf("i915rcu: survived chase (unexpected)\n");
		} else {
			kprintf("i915rcu: [7] mode 1: would read *(u32 *)"
			    "(engine+0x%x) at 0x%016jx (SKIPPED - no fault in "
			    "log mode)\n", CHASE_OFF,
			    (uintmax_t)(uintptr_t)engine + CHASE_OFF);
		}
	}
	crit_exit();				/* rcu_read_unlock */

	kfree(unrelated, M_I915H);
	active_req = NULL;
	return (0);
}

static int
run_sysctl(SYSCTL_HANDLER_ARGS)
{
	int error, v = 0;

	error = sysctl_handle_int(oidp, &v, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if (v == 1 || v == 2)
		run_replica(v);
	return (0);
}

static SYSCTL_NODE(_kern, OID_AUTO, i915rcu, CTLFLAG_RW, 0,
    "i915 rcu replica");
SYSCTL_PROC(_kern_i915rcu, OID_AUTO, run, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, run_sysctl, "I", "1=log-only 2=chase(panic)");

static int
i915rcu_modevent(module_t mod, int type, void *data)
{
	int error = 0;

	switch (type) {
	case MOD_LOAD:
		kprintf("i915rcu: loaded (sysctl kern.i915rcu.run=N)\n");
		break;
	case MOD_UNLOAD:
		break;
	default:
		break;
	}
	return (error);
}

static struct moduledata i915_rcu_harness_mod = { "i915_rcu_harness", i915rcu_modevent, NULL };
DECLARE_MODULE(i915_rcu_harness, i915_rcu_harness_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
MODULE_VERSION(i915_rcu_harness, 1);