--- a/sys/vm/vm_pager.c +++ b/sys/vm/vm_pager.c @@ -156,7 +156,8 @@ &devicepagerops, /* OBJT_DEVICE */ &devicepagerops, /* OBJT_MGTDEVICE */ &physpagerops, /* OBJT_PHYS */ - &deadpagerops /* OBJT_DEAD */ + &deadpagerops, /* OBJT_DEAD */ + &deadpagerops /* OBJT_MARKER (list-scan marker; never paged) */ }; /* @@ -441,8 +442,34 @@ } TAILQ_REMOVE(&bswlist_raw[iter], bp, b_freelist); atomic_add_int(&pbuf_raw_count, -1); - if (pfreecnt) - atomic_add_int(pfreecnt, -1); + if (pfreecnt) { + int c; + + /* + * Atomically reserve against pfreecnt. + * The gate at the top of the loop is + * unlocked and the bucket spinlock is + * per-bucket, so a plain decrement here + * can race multiple CPUs past a count of 1 + * and drive the counter negative. + */ + for (;;) { + c = *(volatile int *)pfreecnt; + if (c <= 0) + break; + if (atomic_cmpset_int(pfreecnt, + c, c - 1)) + break; + } + if (c <= 0) { + /* lost the race, put it back */ + TAILQ_INSERT_HEAD(&bswlist_raw[iter], + bp, b_freelist); + atomic_add_int(&pbuf_raw_count, 1); + spin_unlock(&bswspin_raw[iter]); + continue; + } + } spin_unlock(&bswspin_raw[iter]); initpbuf(bp); @@ -486,8 +513,25 @@ } TAILQ_REMOVE(&bswlist_kva[iter], bp, b_freelist); atomic_add_int(&pbuf_kva_count, -1); - if (pfreecnt) - atomic_add_int(pfreecnt, -1); + if (pfreecnt) { + int c; + + for (;;) { + c = *(volatile int *)pfreecnt; + if (c <= 0) + break; + if (atomic_cmpset_int(pfreecnt, + c, c - 1)) + break; + } + if (c <= 0) { + TAILQ_INSERT_HEAD(&bswlist_kva[iter], + bp, b_freelist); + atomic_add_int(&pbuf_kva_count, 1); + spin_unlock(&bswspin_kva[iter]); + continue; + } + } spin_unlock(&bswspin_kva[iter]); initpbuf(bp); @@ -535,8 +579,25 @@ } TAILQ_REMOVE(&bswlist_mem[iter], bp, b_freelist); atomic_add_int(&pbuf_mem_count, -1); - if (pfreecnt) - atomic_add_int(pfreecnt, -1); + if (pfreecnt) { + int c; + + for (;;) { + c = *(volatile int *)pfreecnt; + if (c <= 0) + break; + if (atomic_cmpset_int(pfreecnt, + c, c - 1)) + break; + } + if (c <= 0) { + TAILQ_INSERT_HEAD(&bswlist_mem[iter], + bp, b_freelist); + atomic_add_int(&pbuf_mem_count, 1); + spin_unlock(&bswspin_mem[iter]); + continue; + } + } spin_unlock(&bswspin_mem[iter]); initpbuf(bp); @@ -575,7 +636,24 @@ } TAILQ_REMOVE(&bswlist_raw[iter], bp, b_freelist); atomic_add_int(&pbuf_raw_count, -1); - atomic_add_int(pfreecnt, -1); + { + int c; + + for (;;) { + c = *(volatile int *)pfreecnt; + if (c <= 0) + break; + if (atomic_cmpset_int(pfreecnt, c, c - 1)) + break; + } + if (c <= 0) { + TAILQ_INSERT_HEAD(&bswlist_raw[iter], bp, + b_freelist); + atomic_add_int(&pbuf_raw_count, 1); + spin_unlock(&bswspin_raw[iter]); + continue; + } + } spin_unlock(&bswspin_raw[iter]); @@ -607,7 +685,24 @@ } TAILQ_REMOVE(&bswlist_kva[iter], bp, b_freelist); atomic_add_int(&pbuf_kva_count, -1); - atomic_add_int(pfreecnt, -1); + { + int c; + + for (;;) { + c = *(volatile int *)pfreecnt; + if (c <= 0) + break; + if (atomic_cmpset_int(pfreecnt, c, c - 1)) + break; + } + if (c <= 0) { + TAILQ_INSERT_HEAD(&bswlist_kva[iter], bp, + b_freelist); + atomic_add_int(&pbuf_kva_count, 1); + spin_unlock(&bswspin_kva[iter]); + continue; + } + } spin_unlock(&bswspin_kva[iter]);