/*
 * DF-3093 racer v3: barrier-synchronized concurrent FAT allocation.
 *
 * MSDOSFS_LOCK_MP()/MSDOSFS_UNLOCK_MP() expand to NOTHING
 * (sys/vfs/msdosfs/msdosfsmount.h:216-218) and struct msdosfsmount has
 * no lock member, so clusteralloc()/chainalloc()/usemap_alloc() and the
 * pm_inusemap scan run unsynchronized on SMP.
 *
 * v3 (lessons from v1/v2 attempts):
 *   - each child is pinned to its own CPU via usched_set(2) (forked
 *     children otherwise timeshare one CPU and never truly overlap);
 *   - the barrier is one go-word per child, each on its own cache line,
 *     written sequentially by the parent (a single shared spin word
 *     serializes the wakeups through one bouncing cache line);
 *   - large per-write cluster counts (default 64 MiB = 16384 clusters
 *     on a 4 KiB-cluster FAT32): chainlength() then READS 512 bitmap
 *     words (~microseconds) before chainalloc() starts SETTING bits,
 *     which scales the read-vs-set race window into the microsecond
 *     range instead of ~100 ns;
 *   - target files are pre-created; children pre-open their fd so the
 *     only post-barrier kernel work is the write(2) itself;
 *   - distinct fill byte per child ('A'+i) so cross-linked clusters
 *     show as foreign bytes; the offline FAT in-degree scanner
 *     (fat_scan.c) is the primary detector.
 *
 * build: cc -O2 -o race2 race2.c
 * usage: ./race2 <mnt> <nprocs> <rounds> <chunkbytes> [pin=1]
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include <sys/types.h>
#include <sys/usched.h>
#include <sys/wait.h>
#include <sys/mman.h>

#define MAXP 32

struct goent {
	volatile unsigned int v;
	char pad[60];
};

struct shm {
	struct goent go[MAXP];		/* per-child barrier words */
	volatile unsigned int ready[MAXP];
	volatile long wrv[MAXP];	/* write() retval */
	volatile int werr[MAXP];	/* errno if short */
	volatile long long trel[MAXP];	/* ns timestamp at release */
	volatile long long tdone[MAXP];	/* ns at write() return */
};

static struct shm *shm;

static long long
nsnow(void)
{
	struct timespec ts;

	clock_gettime(CLOCK_MONOTONIC, &ts);
	return ((long long)ts.tv_sec * 1000000000LL + ts.tv_nsec);
}

int
main(int argc, char **argv)
{
	const char *mnt;
	int nprocs, rounds, pin;
	long chunk;
	int i, r, fd;
	char path[512];
	char *buf;
	pid_t pids[MAXP];

	if (argc != 5 && argc != 6) {
		fprintf(stderr,
		    "usage: %s mnt nprocs rounds chunkbytes [pin=1]\n",
		    argv[0]);
		return (1);
	}
	mnt = argv[1];
	nprocs = atoi(argv[2]);
	rounds = atoi(argv[3]);
	chunk = strtol(argv[4], NULL, 0);
	pin = (argc == 6) ? atoi(argv[5]) : 1;
	if (nprocs < 2 || nprocs > MAXP || rounds < 1 || chunk < 512) {
		fprintf(stderr, "bad args\n");
		return (1);
	}

	shm = mmap(NULL, sizeof(*shm), PROT_READ | PROT_WRITE,
	    MAP_ANON | MAP_SHARED, -1, 0);
	if (shm == MAP_FAILED) { perror("mmap"); return (1); }
	memset((void *)shm, 0, sizeof(*shm));

	/* pre-create every round's files, serialized, before any racing */
	for (r = 0; r < rounds; r++) {
		for (i = 0; i < nprocs; i++) {
			snprintf(path, sizeof(path), "%s/f%dr%d", mnt, r, i);
			fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
			if (fd < 0) { perror(path); return (1); }
			close(fd);
		}
	}

	buf = malloc(chunk);

	for (i = 0; i < nprocs; i++) {
		pid_t pid = fork();
		if (pid == 0) {
			if (pin) {
				int cpuid = i;
				if (usched_set(0, USCHED_SET_CPU, &cpuid,
				    sizeof(cpuid)) != 0)
					perror("usched_set");
			}
			memset(buf, 'A' + (i % 26), chunk);
			for (r = 0; r < rounds; r++) {
				long long t0;
				ssize_t n;

				snprintf(path, sizeof(path), "%s/f%dr%d",
				    mnt, r, i);
				fd = open(path, O_WRONLY);
				if (fd < 0) {
					shm->wrv[i * 8 + r % 8] =
					    -1000 - errno;
					_exit(1);
				}
				__sync_synchronize();
				shm->ready[i] = r + 1;
				while (shm->go[i].v < (unsigned)(r + 1))
					;
				__sync_synchronize();
				t0 = nsnow();
				shm->trel[i] = t0;
				n = write(fd, buf, chunk);
				shm->tdone[i] = nsnow();
				shm->wrv[i] = (long)n;
				if (n != chunk)
					shm->werr[i] = errno;
				close(fd);
			}
			_exit(0);
		}
		pids[i] = pid;
	}

	/* parent: for each round, wait for readiness then fire */
	for (r = 0; r < rounds; r++) {
		int spin = 0;
		for (;;) {
			int done = 1;
			for (i = 0; i < nprocs; i++)
				if (shm->ready[i] < (unsigned)(r + 1)) {
					done = 0;
					break;
				}
			if (done)
				break;
			usleep(50);
			if (++spin > 4000000) {
				fprintf(stderr, "barrier timeout r=%d\n", r);
				return (1);
			}
		}
		__sync_synchronize();
		for (i = 0; i < nprocs; i++)
			shm->go[i].v = r + 1;	/* FIRE (per line) */
	}

	/* reap ALL children, then report (avoids reporting slots of
	 * children that have not finished yet) */
	for (i = 0; i < nprocs; i++) {
		int st;
		waitpid(pids[i], &st, 0);
		if (!WIFEXITED(st) || WEXITSTATUS(st) != 0)
			fprintf(stderr, "child %d status %#x\n", i, st);
	}

	/* timing report for the LAST round */
	{
		long long minrel = 0;
		for (i = 0; i < nprocs; i++)
			if (shm->trel[i] &&
			    (minrel == 0 || shm->trel[i] < minrel))
				minrel = shm->trel[i];
		for (i = 0; i < nprocs; i++)
			printf("  child %d: rel+%lld ns  write=%ld%s  "
			    "dur=%lld us\n", i,
			    shm->trel[i] ? shm->trel[i] - minrel : -1,
			    shm->wrv[i],
			    shm->werr[i] ? strerror(shm->werr[i]) : "",
			    shm->tdone[i] && shm->trel[i] ?
			    (shm->tdone[i] - shm->trel[i]) / 1000 : -1);
	}

	/* in-mount foreign-byte check */
	{
		unsigned long foreign_total = 0;
		char *rb = malloc(chunk);
		for (r = 0; r < rounds; r++) {
			for (i = 0; i < nprocs; i++) {
				ssize_t n, k;
				unsigned long foreign = 0;
				snprintf(path, sizeof(path), "%s/f%dr%d",
				    mnt, r, i);
				fd = open(path, O_RDONLY);
				if (fd < 0)
					continue;
				n = read(fd, rb, chunk);
				for (k = 0; k < n; k++)
					if ((unsigned char)rb[k] !=
					    (unsigned char)('A' + (i % 26)))
						foreign++;
				if (foreign) {
					printf("  r%d p%d: %lu foreign bytes "
					    "<== CROSS-LINK EVIDENCE\n",
					    r, i, foreign);
					foreign_total += foreign;
				}
				close(fd);
			}
		}
		free(rb);
		printf("race2: %d procs x %d rounds x %ldB, "
		    "foreign bytes total: %lu%s\n",
		    nprocs, rounds, chunk, foreign_total,
		    foreign_total ? "  <== REPRODUCED" : "");
	}
	return (0);
}