/*
 * DF-4684 / DF-4685 PoC — amdgpu_fence.c lockless dma_fence_get_rcu
 * refcount-resurrection -> double call_rcu -> double free, and the
 * rcu_barrier() no-op module-unload hazard.
 *
 * This module replicates, line for line, the exact code path of
 * sys/dev/drm/amd/amdgpu/amdgpu_fence.c using the DRM port's OWN
 * primitives (dma_fence_init/signal/release, kref, call_rcu delay-line,
 * kmem_cache shim) — the same code amdgpu.ko executes.  The only thing
 * scheduled "by hand" is the interleaving (on real hardware the fence
 * IRQ preempting the wait_empty reader between its pointer load and its
 * kref_get produces exactly this order).
 *
 * kern.df4684_run=1  DF-4684: wait_empty resurrection -> DOUBLE call_rcu
 *                    on one rcu_head -> kfree(P) twice (alias check).
 * kern.df4684_run=2  DF-4685: enqueue a call_rcu whose callback lives in
 *                    module text, call the port's rcu_barrier() (a NO-OP,
 *                    rcutree.h:30), then the operator kldunloads the module:
 *                    the delay line fires the callback ~1s later into
 *                    unmapped module text -> kernel panic.
 */
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/sysctl.h>
#include <sys/lock.h>
#include <sys/module.h>

#include <linux/slab.h>
#include <linux/kref.h>
#include <linux/dma-fence.h>
#include <linux/rcupdate.h>

/* ------------------------------------------------------------------ */
/* The "amdgpu fence engine" replica: emit / process / release / free  */
/* mirroring amdgpu_fence.c:134-170, 222-268, 631-649                  */
/* ------------------------------------------------------------------ */

struct df4684_fence {
	struct dma_fence base;			/* amdgpu_fence.c:50-55 */
};

static struct kmem_cache *df4684_slab;		/* amdgpu_fence_slab :57 */
static struct lock df4684_drv_lock;		/* ring->fence_drv.lock */
static struct dma_fence *df4684_slot;		/* ring->fence_drv.fences[i] */
static int df4684_release_count;		/* call_rcu enqueues */
static int df4684_free_count;			/* delay-line frees executed */

/* == amdgpu_fence_free, amdgpu_fence.c:631-636 (delay-line callback) */
static void
df4684_fence_free(struct rcu_head *rcu)
{
	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
	struct df4684_fence *fence =
	    container_of(f, struct df4684_fence, base);

	++df4684_free_count;
	kprintf("df4684: [delay-line fired #%d] amdgpu_fence_free equivalent:"
	    " kmem_cache_free(%p)\n", df4684_free_count, fence);
	kmem_cache_free(df4684_slab, fence);
}

/* == amdgpu_fence_release, amdgpu_fence.c:646-649 */
static void
df4684_fence_release(struct dma_fence *f)
{
	++df4684_release_count;
	kprintf("df4684: release #%d: refcount hit 0 -> call_rcu(&f->rcu=%p,"
	    " free)\n", df4684_release_count, &f->rcu);
	call_rcu(&f->rcu, df4684_fence_free);
}

static const char *
df4684_driver_name(struct dma_fence *fence)
{
	return ("df4684");
}

static const char *
df4684_timeline_name(struct dma_fence *f)
{
	return ("df4684-ring");
}

static const struct dma_fence_ops df4684_fence_ops = {
	.get_driver_name	= df4684_driver_name,
	.get_timeline_name	= df4684_timeline_name,
	.release		= df4684_fence_release,	/* amdgpu_fence.c:655 */
};

/* ------------------------------------------------------------------ */
/* DF-4684: the amdgpu_fence_wait_empty interleave, deterministically  */
/* ------------------------------------------------------------------ */

static int
df4684_run_mode1(void)
{
	struct df4684_fence *fe;
	struct dma_fence *fence, *loaded, *writer_f;
	void *a, *b;
	long r;

	kprintf("df4684: === mode 1: wait_empty lockless get_rcu ===\n");

	df4684_slab = kmem_cache_create("df4684_fence",
	    sizeof(struct df4684_fence), 0, 0, NULL);
	if (df4684_slab == NULL)
		return (ENOMEM);

	/* == amdgpu_fence_emit, amdgpu_fence.c:142-169 */
	fe = kmem_cache_alloc(df4684_slab, GFP_KERNEL);
	if (fe == NULL)
		return (ENOMEM);
	dma_fence_init(&fe->base, &df4684_fence_ops, &df4684_drv_lock,
	    0x4684, 1);				/* refcount = 1 (caller) */
	fence = &fe->base;
	rcu_assign_pointer(df4684_slot, dma_fence_get(fence)); /* array ref=2 */
	dma_fence_put(fence);			/* caller drops -> 1: array-only */
	fence = NULL;

	kprintf("df4684: fence=%p refcount=%u (array ref only — the steady"
	    " state of a completed, unprocessed fence)\n",
	    fe, kref_read(&fe->base.refcount));

	/*
	 * == amdgpu_fence_wait_empty, amdgpu_fence.c:305-311
	 * (running in the PM/teardown thread when the fence-completion
	 *  IRQ preempts it right here)
	 */
	rcu_read_lock();			/* == preempt_disable() */
	loaded = rcu_dereference(df4684_slot);	/* :306 load P */
	kprintf("df4684: reader :306 rcu_dereference(slot) = %p\n", loaded);
	kprintf("df4684: *** fence IRQ fires NOW, preempting the reader ***\n");

	/* == amdgpu_fence_process, amdgpu_fence.c:244-265 (IRQ context) */
	writer_f = rcu_dereference_protected(df4684_slot, 1);	/* :252 */
	RCU_INIT_POINTER(df4684_slot, NULL);			/* :253 */
	if (writer_f != NULL) {
		r = dma_fence_signal(writer_f);			/* :258 */
		kprintf("df4684: writer :258 dma_fence_signal -> %ld\n", r);
		dma_fence_put(writer_f);			/* :264 1->0
							 * -> release
							 * -> call_rcu #1 */
	}

	/* reader resumes at :307 — the port's dma_fence_get_rcu is an
	 * UNCONDITIONAL kref_get (dma-fence.h:97-103): it resurrects the
	 * already-released fence 0 -> 1 and never returns NULL. */
	fence = dma_fence_get_rcu(loaded);			/* :307 */
	kprintf("df4684: reader :307 dma_fence_get_rcu(%p) = %p,"
	    " refcount resurrected to %u\n",
	    loaded, fence, kref_read(&loaded->refcount));
	if (fence == NULL) {			/* never taken on this port */
		rcu_read_unlock();
		kprintf("df4684: get_rcu refused the dead fence (FIXED"
		    " header) -> wait_empty returns 0, no put\n");
		return (0);
	}
	rcu_read_unlock();			/* :311 */

	/* :313-314 */
	r = dma_fence_wait(fence, false);
	kprintf("df4684: reader :313 dma_fence_wait -> %ld (signaled)\n", r);
	dma_fence_put(fence);			/* on this port: saturated
						 * refcount means this put
						 * does NOT hit zero — with
						 * plain kref_get semantics
						 * (or a fixed refcount_inc)
						 * it would enqueue a SECOND
						 * call_rcu on the same head */
	kprintf("df4684: reader :314 dma_fence_put done, refcount now %d\n",
	    kref_read(&fence->refcount));

	kprintf("df4684: sleeping 3s while the >=1s delay line"
	    " (linux_rcu.c rcu_timer) fires queued entries...\n");
	tsleep(&df4684_run_mode1, 0, "df4684", 3 * hz);

	/* consequence check */
	a = kmem_cache_alloc(df4684_slab, GFP_KERNEL);
	b = kmem_cache_alloc(df4684_slab, GFP_KERNEL);
	kprintf("df4684: RESULT: releases=%d frees=%d", df4684_release_count,
	    df4684_free_count);
	if (df4684_release_count == 1 && df4684_free_count == 1) {
		kprintf(" — the resurrected fence was used after its final"
		    " release, but the port's refcount_inc SATURATION"
		    " (refcount.h:53-60) kept the reader's put from reaching"
		    " zero, so the second release/double-free tail is BLOCKED"
		    " on this port (upstream bails at :307 instead)\n");
	} else if (df4684_free_count >= 2) {
		kprintf(" — DOUBLE FREE\n");
	} else {
		kprintf("\n");
	}
	kprintf("df4684: post-run allocs: a=%p b=%p (%s)\n", a, b,
	    (a == b) ? "ALIASED" : "distinct");
	kprintf("df4684: === mode 1 complete ===\n");
	return (0);
}

/* ------------------------------------------------------------------ */
/* DF-4685: rcu_barrier() no-op -> delayed callback into freed text    */
/* ------------------------------------------------------------------ */

static struct rcu_head df4684_dummy_rcu;

static void
df4684_dummy_cb(struct rcu_head *rcu)
{
	/* this instruction lives in df4684.ko's mapped text */
	kprintf("df4684: dummy RCU callback executed (module text %p"
	    " still mapped)\n", df4684_dummy_cb);
}

static int
df4684_run_mode2(void)
{
	kprintf("df4684: === mode 2: rcu_barrier() no-op vs module unload ===\n");
	call_rcu(&df4684_dummy_rcu, df4684_dummy_cb);
	kprintf("df4684: call_rcu enqueued (callback %p fires in >=1s)\n",
	    df4684_dummy_cb);
	/* == amdgpu_fence_slab_fini, amdgpu_fence.c:69-73 — the barrier is
	 * supposed to drain the delay line before module text disappears */
	rcu_barrier();			/* NO-OP: rcutree.h:30 */
	kprintf("df4684: rcu_barrier() returned (no-op on this port) —"
	    " kldunload NOW; the pending callback will execute into"
	    " unmapped module text ~1s later\n");
	kprintf("df4684: === mode 2 armed — run: kldunload df4684; sleep 3 ===\n");
	return (0);
}

/* ------------------------------------------------------------------ */

static int
df4684_run_sysctl(SYSCTL_HANDLER_ARGS)
{
	int mode, error;

	mode = 0;
	error = SYSCTL_IN(req, &mode, sizeof(mode));
	if (error != 0 || req->newptr == NULL)
		return (error);

	switch (mode) {
	case 1:
		return (df4684_run_mode1());
	case 2:
		return (df4684_run_mode2());
	default:
		return (EINVAL);
	}
}

SYSCTL_PROC(_kern, OID_AUTO, df4684_run, CTLTYPE_INT | CTLFLAG_RW,
    0, 0, df4684_run_sysctl, "I", "DF-4684/DF-4685 trigger: 1=double-free 2=unload-hazard");

static int
df4684_load(module_t mod, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		lockinit(&df4684_drv_lock, "df4684l", 0, LK_CANRECURSE);
		kprintf("df4684: loaded (amdgpu_fence replica)\n");
		return (0);
	case MOD_UNLOAD:
		/* Intentionally does NOT drain pending delay-line entries:
		 * that is what the (broken) rcu_barrier() contract was
		 * supposed to guarantee upstream of module unload. */
		kprintf("df4684: unloading\n");
		return (0);
	default:
		return (0);
	}
}

static moduledata_t df4684_mod = {
	"df4684",
	df4684_load,
	NULL
};

DECLARE_MODULE(df4684, df4684_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
MODULE_VERSION(df4684, 1);
/* NB: deliberately NO MODULE_DEPEND on drm: DF's linker does not
 * refcount dependencies (kern_linker.c:574 linker_file_add_dependancy
 * never bumps dep->refs), so a declared dep makes kldunload tear down
 * drm.ko itself (which wedges headless).  drm.ko must be kldload(8)ed
 * before this module for symbol resolution; symbols used: call_rcu,
 * dma_fence_init/signal/wait_timeout (linux_rcu.c, linux_fence.c). */