/* * DF-2214 verbatim-code rig - HEAD * Target: sys/dev/drm/radeon/rv770_dma.c:42-94 rv770_copy_dma() * Claim under test: size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4 * computed in 32-bit unsigned arithmetic wraps for * num_gpu_pages >= 2^20 (4 GiB-class TTM moves), so the DMA * engine is told to copy only a fraction (or nothing) while * radeon_fence_emit() still signals success. * * This head provides the minimal FABRICATED boundary state the verbatim * production function walks: the real types it names, real constants * (RADEON_GPU_PAGE_SHIFT, DMA_PACKET/DMA_PACKET_COPY from rv770d.h, real DMA * ring geometry from rv770.c:1968), and shim implementations of the radeon * helpers that replicate the observable semantics of the production bodies * (each cited). All ring writes land in a kmalloc'd ring buffer bracketed by * canary zones so the "no host kernel memory is corrupted" negative claim is * tested empirically, not just asserted. * * The production function body itself is concatenated UNMODIFIED (byte range * lines 42..94 of the tree file) by build.sh - see the sha256 checks there. */ #include /* howmany */ #include #include /* kprintf */ #include /* SI_SUB_* */ #include #include #include #include /* no : sys/systm.h provides the fortified memset macro; a * prototype would collide with it in kernel context */ /* ------------------------------------------------------------------ */ /* Types the verbatim body names (fabricated minimal, member paths */ /* exactly as dereferenced by the production code) */ /* ------------------------------------------------------------------ */ typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; /* bool comes from sys/stdint.h (typedef _Bool bool) in this environment */ #ifndef __bool_true_false_are_defined #define false 0 #define true 1 #define __bool_true_false_are_defined 1 #endif struct reservation_object { int rig_dummy; }; struct radeon_device; struct radeon_fence { struct radeon_device *rdev; int ring; u64 seq; /* radeon_fence.c:117-127 shape */ }; /* real struct radeon_sync holds 2 semaphore ptrs + a fence (radeon.h:619-622); * the verbatim body only passes &sync around - shape is irrelevant, calls are */ struct radeon_sync { void *s[2]; struct radeon_fence *f; }; struct radeon_ring { int idx; /* radeon.h ring block */ u32 *ring; /* CPU mapping of ring buffer */ unsigned ring_size; /* bytes (radeon_ring.c:106) */ u32 wptr; /* radeon.h:2708-2717 state */ u32 wptr_old; u32 ptr_mask; /* radeon_ring.c:410 */ u32 align_mask; /* radeon_ring.c:114 */ int count_dw; /* radeon.h:2710/2715 */ int ring_free_dw; int ready; /* radeon_ttm.c move gate */ }; struct radeon_asic { struct { int dma_ring_index; } copy; /* radeon.h asic->copy */ }; #define RADEON_NUM_RINGS 8 /* radeon.h:161 */ struct radeon_device { const struct radeon_asic *asic; struct radeon_ring ring[RADEON_NUM_RINGS]; u64 sync_seq[RADEON_NUM_RINGS]; /* radeon.h:369 */ struct lock ring_lock; /* radeon_ring.c:143 */ }; /* ------------------------------------------------------------------ */ /* Real constants/macros (verbatim from the cited tree lines) */ /* ------------------------------------------------------------------ */ #define RADEON_GPU_PAGE_SHIFT 12 /* radeon.h:645 */ #define R600_RING_TYPE_DMA_INDEX 3 /* radeon.h:149 */ /* rv770d.h:663-666 verbatim */ #define DMA_PACKET(cmd, t, s, n) ((((cmd) & 0xF) << 28) | \ (((t) & 0x1) << 23) | \ (((s) & 0x1) << 22) | \ (((n) & 0xFFFF) << 0)) /* rv770d.h:669 */ #define DMA_PACKET_COPY 0x3 /* sys/dev/drm/include/linux/kernel.h:59 */ #define DIV_ROUND_UP(x, y) howmany(x, y) /* linux compat semantics used by the body */ #define upper_32_bits(n) ((u32)(((n) >> 16) >> 16)) #define MAX_ERRNO 4096L #define ERR_PTR(e) ((void *)(intptr_t)(e)) #define IS_ERR_VALUE(x) ((unsigned long)(x) >= (unsigned long)-MAX_ERRNO) #define IS_ERR(p) IS_ERR_VALUE((intptr_t)(p)) /* ------------------------------------------------------------------ */ /* Rig observation state */ /* ------------------------------------------------------------------ */ struct rig_stats { unsigned long writes; /* total radeon_ring_write calls */ unsigned copy_packets; /* DMA_PACKET_COPY headers seen */ u64 sum_chunk_dw; /* sum of packet n fields */ u32 max_chunk, min_chunk; u32 first_pkt; /* first packet header dword */ u32 first5[5], last5[5]; /* raw ring dwords */ int fence_emits; u64 fence_seq; int fence_ring; int lock_rc; unsigned attempted_ndw; /* ndw passed to radeon_ring_lock */ unsigned committed; unsigned undone; int underruns; /* count_dw<=0 at write time */ int sync_create, sync_resv, sync_rings, sync_free; int drm_errors; }; static struct rig_stats g_stats; #define DRM_ERROR(fmt, ...) do { g_stats.drm_errors++; \ kprintf("DF2214 [DRM_ERROR] " fmt, ##__VA_ARGS__); } while (0) /* ------------------------------------------------------------------ */ /* Shim: radeon_sync_* (production radeon_sync.c:42,89,137,212) */ /* ------------------------------------------------------------------ */ static void radeon_sync_create(struct radeon_sync *sync) /* :42 */ { memset(sync, 0, sizeof(*sync)); g_stats.sync_create++; } static int radeon_sync_resv(struct radeon_device *rdev, /* :89 */ struct radeon_sync *sync, struct reservation_object *resv, bool shared) { g_stats.sync_resv++; return 0; } static int radeon_sync_rings(struct radeon_device *rdev, /* :137 */ struct radeon_sync *sync, int ring) { g_stats.sync_rings++; return 0; } static void radeon_sync_free(struct radeon_device *rdev, /* :212 */ struct radeon_sync *sync, struct radeon_fence *fence) { g_stats.sync_free++; } /* ------------------------------------------------------------------ */ /* Shim: ring allocation/lock - observable semantics of */ /* radeon_ring.c:107-127 (radeon_ring_alloc) and 140-152 (lock) */ /* ------------------------------------------------------------------ */ static int rig_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw) { /* radeon_ring.c:108-110 */ if (ndw > (ring->ring_size / 4)) return -ENOMEM; /* radeon_ring.c:113-114 */ ndw = (ndw + ring->align_mask) & ~ring->align_mask; ring->count_dw = ndw; ring->wptr_old = ring->wptr; return 0; } static int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw) { int r; g_stats.attempted_ndw = ndw; /* rig observation only */ lockmgr(&rdev->ring_lock, LK_EXCLUSIVE); r = rig_ring_alloc(rdev, ring, ndw); if (r) { lockmgr(&rdev->ring_lock, LK_RELEASE); g_stats.lock_rc = r; return r; } g_stats.lock_rc = 0; return 0; } /* radeon_ring.c:195-213: unlock_commit pads+releases; padding writes 0 in * this boundary state (align_mask==0); observed effect = commit count */ static void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring, bool b) { g_stats.committed++; lockmgr(&rdev->ring_lock, LK_RELEASE); } /* radeon_ring.c:221-226: undo resets wptr to the allocation-time value */ static void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring) { ring->wptr = ring->wptr_old; g_stats.undone++; lockmgr(&rdev->ring_lock, LK_RELEASE); } /* ------------------------------------------------------------------ */ /* Shim: radeon_ring_write - PRODUCTION BODY VERBATIM radeon.h:2710- */ /* 2716, plus trailing rig instrumentation (clearly separated). The */ /* production bytes write ring->ring[wptr++&mask] and count down */ /* count_dw / ring_free_dw; every write stays inside the ring buffer */ /* allocation by the ptr_mask invariant. */ /* ------------------------------------------------------------------ */ static void radeon_ring_write(struct radeon_ring *ring, uint32_t v) { if (ring->count_dw <= 0) DRM_ERROR("radeon: writing more dwords to the ring than expected!\n"); ring->ring[ring->wptr++] = v; ring->wptr &= ring->ptr_mask; ring->count_dw--; ring->ring_free_dw--; /* ---- rig instrumentation below, NOT production bytes ---- */ { u32 n5 = (u32)((g_stats.writes + 1) % 5); g_stats.writes++; if (g_stats.writes <= 5) g_stats.first5[g_stats.writes - 1] = v; g_stats.last5[n5] = v; if (n5 == 1) { /* packet header position in the 5-dword cadence */ u32 cmd = (v >> 28) & 0xF; u32 n = v & 0xFFFF; if (cmd == DMA_PACKET_COPY) { if (g_stats.copy_packets == 0) { g_stats.first_pkt = v; g_stats.min_chunk = 0xFFFFFFFFu; } g_stats.copy_packets++; g_stats.sum_chunk_dw += n; if (n > g_stats.max_chunk) g_stats.max_chunk = n; if (n < g_stats.min_chunk) g_stats.min_chunk = n; } } } } /* ------------------------------------------------------------------ */ /* Shim: radeon_fence_emit - production radeon_fence.c:128-141 */ /* (kmalloc + ++sync_seq[ring]); fence packets consume the +8 slack */ /* reserved by the caller, which the rig records as 0 writes. */ /* ------------------------------------------------------------------ */ static int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring) { struct radeon_fence *f; f = kmalloc(sizeof(*f), M_TEMP, M_WAITOK | M_ZERO); f->rdev = rdev; f->ring = ring; f->seq = ++rdev->sync_seq[ring]; *fence = f; g_stats.fence_emits++; g_stats.fence_seq = f->seq; g_stats.fence_ring = ring; return 0; } struct radeon_fence *rv770_copy_dma(struct radeon_device *rdev, uint64_t src_offset, uint64_t dst_offset, unsigned num_gpu_pages, struct reservation_object *resv) { struct radeon_fence *fence; struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_dw, cur_size_in_dw; int i, num_loops; int r = 0; radeon_sync_create(&sync); size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFF); r = radeon_ring_lock(rdev, ring, num_loops * 5 + 8); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_sync_resv(rdev, &sync, resv, false); radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_dw = size_in_dw; if (cur_size_in_dw > 0xFFFF) cur_size_in_dw = 0xFFFF; size_in_dw -= cur_size_in_dw; radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); radeon_ring_write(ring, dst_offset & 0xfffffffc); radeon_ring_write(ring, src_offset & 0xfffffffc); radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); src_offset += cur_size_in_dw * 4; dst_offset += cur_size_in_dw * 4; } r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); radeon_sync_free(rdev, &sync, fence); return fence; } struct radeon_fence *rv770_copy_dma_fixed(struct radeon_device *rdev, uint64_t src_offset, uint64_t dst_offset, unsigned num_gpu_pages, struct reservation_object *resv) { struct radeon_fence *fence; struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u64 size_in_dw; u32 cur_size_in_dw; int i, num_loops; int r = 0; radeon_sync_create(&sync); size_in_dw = ((u64)num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFF); r = radeon_ring_lock(rdev, ring, num_loops * 5 + 8); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_sync_resv(rdev, &sync, resv, false); radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_dw = size_in_dw; if (cur_size_in_dw > 0xFFFF) cur_size_in_dw = 0xFFFF; size_in_dw -= cur_size_in_dw; radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw)); radeon_ring_write(ring, dst_offset & 0xfffffffc); radeon_ring_write(ring, src_offset & 0xfffffffc); radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); src_offset += cur_size_in_dw * 4; dst_offset += cur_size_in_dw * 4; } r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); radeon_sync_free(rdev, &sync, fence); return fence; } /* * DF-2214 verbatim-code rig - TAIL * Matrix driver + module glue. Runs the UNMODIFIED production function * (rv770_copy_dma, concatenated above from tree lines 42-94) and the FIXED * variant (rv770_copy_dma_fixed, fix.diff applied + renamed) over the exact * input domain that flips the claimed bug and prints, per case: * - the ndw the function tried to reserve (== num_loops*5+8) * - every DMA_PACKET_COPY header emitted into the ring (count, chunk sizes) * - whether a success fence was emitted * - bytes the DMA engine was actually told to move vs bytes requested * - canary integrity around the ring buffer (host-memory-safety check) */ #define CANARY_ZONE 4096 #define CANARY_WORD 0xCAFEBABEu #define RIG_SRC_BASE 0x123456789ABCLL /* nontrivial 44-bit GPU addrs */ #define RIG_DST_BASE 0xFEDCBA987654LL struct rig_case { unsigned pages; unsigned ring_bytes; /* ring buffer geometry */ const char *ring_tag; }; static const struct rig_case rig_cases[] = { /* --- big ring (4 MiB, plenty for every reservation below) --- */ { 0x000FFFFFu, 0x400000u, "big4M" }, /* 2^20-1: last non-wrapping */ { 0x00100000u, 0x400000u, "big4M" }, /* 2^20 : exact wrap to 0 */ { 0x00100001u, 0x400000u, "big4M" }, /* 2^20+1: wraps to 4 KiB */ { 0x00100800u, 0x400000u, "big4M" }, /* 2^20+2048: wraps to 8 MiB */ { 0xC0000000u, 0x400000u, "big4M" }, /* 12 TiB request: total loss*/ /* --- realistic rv770 DMA ring: 64 KiB (rv770.c:1968) --- */ { 0x00008000u, 0x010000u, "real64K" }, /* 128 MiB move: fits, control */ { 0x000FFFFFu, 0x010000u, "real64K" }, /* 4 GiB-4K: loud ENOMEM (fixed/correct) vs silent no-op (buggy) */ { 0x00100000u, 0x010000u, "real64K" }, /* 4 GiB: the money shot */ }; static int rig_canary_ok(u32 *buf, unsigned ring_bytes) { unsigned i, words = CANARY_ZONE / 4; for (i = 0; i < words; i++) if (buf[i] != CANARY_WORD || buf[words + ring_bytes / 4 + i] != CANARY_WORD) return 0; return 1; } static void run_case(int fixed, const struct rig_case *c) { static const struct radeon_asic asic = { .copy = { .dma_ring_index = R600_RING_TYPE_DMA_INDEX }, }; struct radeon_device rdev; struct reservation_object resv; struct radeon_fence *fence; u32 *buf; unsigned i; int err = 0; u64 requested, scheduled; int canary; int is_err_ptr; memset(&rdev, 0, sizeof(rdev)); rdev.asic = &asic; lockinit(&rdev.ring_lock, "df2214lk", 0, 0); /* ring geometry per radeon_ring_init (radeon_ring.c:404-411) */ { struct radeon_ring *ring = &rdev.ring[R600_RING_TYPE_DMA_INDEX]; ring->idx = R600_RING_TYPE_DMA_INDEX; ring->ring_size = c->ring_bytes; ring->ptr_mask = (ring->ring_size / 4) - 1; ring->ring_free_dw = ring->ring_size / 4; ring->align_mask = 0; /* boundary state: no commit padding */ ring->ready = 1; } /* ring buffer bracketed by canary zones */ buf = kmalloc(c->ring_bytes + 2 * CANARY_ZONE, M_TEMP, M_WAITOK | M_ZERO); for (i = 0; i < CANARY_ZONE / 4; i++) { buf[i] = CANARY_WORD; buf[CANARY_ZONE / 4 + c->ring_bytes / 4 + i] = CANARY_WORD; } rdev.ring[R600_RING_TYPE_DMA_INDEX].ring = buf + CANARY_ZONE / 4; memset(&g_stats, 0, sizeof(g_stats)); g_stats.min_chunk = 0xFFFFFFFFu; g_stats.max_chunk = 0; requested = (u64)c->pages << RADEON_GPU_PAGE_SHIFT; /* rig-side 64-bit truth */ if (fixed) fence = rv770_copy_dma_fixed(&rdev, RIG_SRC_BASE, RIG_DST_BASE, c->pages, &resv); else fence = rv770_copy_dma(&rdev, RIG_SRC_BASE, RIG_DST_BASE, c->pages, &resv); is_err_ptr = IS_ERR(fence); if (is_err_ptr) err = (int)(intptr_t)fence; scheduled = g_stats.sum_chunk_dw * 4; canary = rig_canary_ok(buf, c->ring_bytes); kprintf("DF2214 CASE variant=%s pages=0x%08x ring=%s(%u bytes)\n", fixed ? "FIXED" : "BASELINE", c->pages, c->ring_tag, c->ring_bytes); kprintf("DF2214 requested_bytes=%llu (0x%llx) attempted_ndw=%u lock_rc=%d\n", (unsigned long long)requested, (unsigned long long)requested, g_stats.attempted_ndw, g_stats.lock_rc); if (is_err_ptr) { kprintf("DF2214 ERR_PTR(%d): ring reservation refused, ZERO packets, NO fence - loud failure\n", err); kprintf("DF2214 VERDICT: ERR_PTR_FAIL\n"); } else { unsigned loops = g_stats.attempted_ndw ? (g_stats.attempted_ndw - 8) / 5 : 0; kprintf("DF2214 num_loops=%u copy_packets=%u sum_chunk_dw=%llu max_chunk=%u min_chunk=%u ring_writes=%lu\n", loops, g_stats.copy_packets, (unsigned long long)g_stats.sum_chunk_dw, g_stats.max_chunk, g_stats.min_chunk, g_stats.writes); kprintf("DF2214 first_pkt=0x%08x (cmd=%u n=%u) first5=[%08x %08x %08x %08x %08x]\n", g_stats.first_pkt, (g_stats.first_pkt >> 28) & 0xF, g_stats.first_pkt & 0xFFFF, g_stats.first5[0], g_stats.first5[1], g_stats.first5[2], g_stats.first5[3], g_stats.first5[4]); kprintf("DF2214 scheduled_bytes=%llu (%lu.%04lu%% of request) fence_emitted=%d ring=%d seq=%llu\n", (unsigned long long)scheduled, (unsigned long)(scheduled * 100 / (requested ? requested : 1)), (unsigned long)((scheduled * 1000000 / (requested ? requested : 1)) % 10000), g_stats.fence_emits, g_stats.fence_ring, (unsigned long long)g_stats.fence_seq); kprintf("DF2214 integrity: canaries=%s count_dw_final=%d underruns=%d drm_errors=%d sync{c=%d r=%d g=%d f=%d} commit=%u\n", canary ? "INTACT" : "CLOBBERED", rdev.ring[R600_RING_TYPE_DMA_INDEX].count_dw, g_stats.underruns, g_stats.drm_errors, g_stats.sync_create, g_stats.sync_resv, g_stats.sync_rings, g_stats.sync_free, g_stats.committed); if (scheduled == requested) kprintf("DF2214 VERDICT: FULL_COPY\n"); else if (g_stats.fence_emits) kprintf("DF2214 VERDICT: SILENT_TRUNCATION (fence says success, %llu of %llu bytes scheduled)\n", (unsigned long long)scheduled, (unsigned long long)requested); else kprintf("DF2214 VERDICT: INTERNAL_ERROR\n"); } if (!is_err_ptr && fence) kfree(fence, M_TEMP); kfree(buf, M_TEMP); lockuninit(&rdev.ring_lock); } static void rig_matrix(void) { size_t i; kprintf("DF2214 RIG start: kernel-resident verbatim rv770_copy_dma (baseline) + fix.diff variant\n"); for (i = 0; i < sizeof(rig_cases) / sizeof(rig_cases[0]); i++) run_case(0, &rig_cases[i]); for (i = 0; i < sizeof(rig_cases) / sizeof(rig_cases[0]); i++) run_case(1, &rig_cases[i]); kprintf("DF2214 RIG done\n"); } static int df2214_modevent(module_t mod, int type, void *arg) { switch (type) { case MOD_LOAD: rig_matrix(); return 0; case MOD_UNLOAD: return 0; default: return EOPNOTSUPP; } } static moduledata_t df2214_mod = { "df2214_rig", df2214_modevent, NULL }; DECLARE_MODULE(df2214_rig, df2214_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); MODULE_VERSION(df2214_rig, 1);