--- a/sys/kern/vfs_journal.c +++ b/sys/kern/vfs_journal.c @@ -111,6 +111,21 @@ MALLOC_DEFINE(M_JOURNAL, "journal", "Journaling structures"); MALLOC_DEFINE(M_JFIFO, "journal-fifo", "Journal FIFO"); +/* + * DF-2747: the memory FIFO reservation protocol (journal_reserve(), + * journal_extend(), journal_abort(), journal_commit() and the worker + * thread index updates) manipulates jo->fifo.{w,x,r}index and + * jo->transid with unsynchronized read-modify-write cycles. Journal + * VOP shims on a journaled mount execute concurrently on SMP, so + * concurrent VOPs overlap reservations and corrupt the raw record + * chain that journal_wthread() parses. Serialize the critical + * sections with a global spinlock until the layer gains proper + * per-journal locking (journaling is a rarely-configured feature, + * contention is acceptable). + */ +static struct spinlock journal_fifo_lock = + SPINLOCK_INITIALIZER(journal_fifo_lock, "journal_fifo_lock"); + void journal_create_threads(struct journal *jo) { @@ -211,14 +226,19 @@ * help it. */ if (rawp->streamid == JREC_STREAMID_PAD) { - if ((jo->flags & MC_JOURNAL_WANT_FULLDUPLEX) == 0) { + spin_lock(&journal_fifo_lock); /* DF-2747 */ + if ((jo->flags & (MC_JOURNAL_WANT_FULLDUPLEX | + MC_JOURNAL_RACTIVE)) != + (MC_JOURNAL_WANT_FULLDUPLEX | MC_JOURNAL_RACTIVE)) { + /* DF-2748: half duplex, or fullduplex w/ dead ack thread */ if (jo->fifo.rindex == jo->fifo.xindex) { jo->fifo.xindex += (rawp->recsize + 15) & ~15; jo->total_acked += (rawp->recsize + 15) & ~15; } } jo->fifo.rindex += (rawp->recsize + 15) & ~15; - jo->total_acked += bytes; + jo->total_acked += (rawp->recsize + 15) & ~15; /* DF-2749: was: += bytes */ + spin_unlock(&journal_fifo_lock); /* DF-2747 pad-skip */ KKASSERT(jo->fifo.windex - jo->fifo.rindex >= 0); continue; } @@ -258,7 +278,9 @@ * XXX two-way acknowledgement stream in the return direction / xindex */ bytes = res; + spin_lock(&journal_fifo_lock); /* DF-2747 */ jo->fifo.rindex += bytes; + spin_unlock(&journal_fifo_lock); /* DF-2747 */ error = fp_write(jo->fp, jo->fifo.membase + ((jo->fifo.rindex - bytes) & jo->fifo.mask), @@ -275,12 +297,20 @@ * advance xindex, otherwise the rjournal thread is responsible for * advancing xindex. */ - if ((jo->flags & MC_JOURNAL_WANT_FULLDUPLEX) == 0) { + if ((jo->flags & (MC_JOURNAL_WANT_FULLDUPLEX | + MC_JOURNAL_RACTIVE)) != + (MC_JOURNAL_WANT_FULLDUPLEX | MC_JOURNAL_RACTIVE)) { + /* DF-2748: half duplex, or fullduplex w/ dead ack thread */ + spin_lock(&journal_fifo_lock); /* DF-2747 */ jo->fifo.xindex += bytes; + spin_unlock(&journal_fifo_lock); /* DF-2747 */ jo->total_acked += bytes; } KKASSERT(jo->fifo.windex - jo->fifo.rindex >= 0); - if ((jo->flags & MC_JOURNAL_WANT_FULLDUPLEX) == 0) { + if ((jo->flags & (MC_JOURNAL_WANT_FULLDUPLEX | + MC_JOURNAL_RACTIVE)) != + (MC_JOURNAL_WANT_FULLDUPLEX | MC_JOURNAL_RACTIVE)) { + /* DF-2748: half duplex, or fullduplex w/ dead ack thread */ if (jo->flags & MC_JOURNAL_WWAIT) { jo->flags &= ~MC_JOURNAL_WWAIT; /* XXX hysteresis */ wakeup(&jo->fifo.windex); @@ -380,7 +410,9 @@ (long long)rawp->transid, (long long)transid); #endif + spin_lock(&journal_fifo_lock); /* DF-2747 */ jo->fifo.xindex += (rawp->recsize + 15) & ~15; + spin_unlock(&journal_fifo_lock); /* DF-2747 */ jo->total_acked += (rawp->recsize + 15) & ~15; if (jo->flags & MC_JOURNAL_WWAIT) { jo->flags &= ~MC_JOURNAL_WWAIT; /* XXX hysteresis */ @@ -394,7 +426,9 @@ (long long)rawp->transid, (long long)transid); #endif + spin_lock(&journal_fifo_lock); /* DF-2747 */ jo->fifo.xindex += (rawp->recsize + 15) & ~15; + spin_unlock(&journal_fifo_lock); /* DF-2747 */ jo->total_acked += (rawp->recsize + 15) & ~15; if (jo->flags & MC_JOURNAL_WWAIT) { jo->flags &= ~MC_JOURNAL_WWAIT; /* XXX hysteresis */ @@ -463,8 +497,10 @@ { int avail; + spin_lock(&journal_fifo_lock); /* DF-2747 */ avail = jo->fifo.size - (jo->fifo.windex - jo->fifo.xindex); KKASSERT(avail >= 0); + spin_unlock(&journal_fifo_lock); /* DF-2747 */ if ((avail < (jo->fifo.size >> 1)) || (jo->flags & MC_JOURNAL_WWAIT)) wakeup(&jo->fifo); } @@ -524,6 +560,7 @@ * Also, since all fifo ops are 16-byte aligned, we can check * the size before calculating the aligned size. */ + spin_lock(&journal_fifo_lock); /* DF-2747 */ availtoend = jo->fifo.size - (jo->fifo.windex & jo->fifo.mask); KKASSERT((availtoend & 15) == 0); if (bytes > availtoend) @@ -544,6 +581,7 @@ /* XXX MC_JOURNAL_STOP_IMM */ jo->flags |= MC_JOURNAL_WWAIT; ++jo->fifostalls; + spin_unlock(&journal_fifo_lock); /* DF-2747: stall */ tsleep(&jo->fifo.windex, 0, "jwrite", 0); continue; } @@ -583,6 +621,7 @@ */ cpu_sfence(); jo->fifo.windex += (req + 15) & ~15; + spin_unlock(&journal_fifo_lock); /* DF-2747 */ *rawpp = rawp; return(rawp + 1); } @@ -620,6 +659,7 @@ int wbase; void *rptr; + spin_lock(&journal_fifo_lock); /* DF-2747 */ *newstreamrecp = 0; rawp = *rawpp; osize = (rawp->recsize + 15) & ~15; @@ -631,6 +671,7 @@ * the record size. */ if (nsize == osize) { + spin_unlock(&journal_fifo_lock); /* DF-2747 ext-1 */ rawp->recsize += bytes; return((char *)(rawp + 1) + truncbytes); } @@ -647,11 +688,13 @@ KKASSERT((avail & 15) == 0); if (nsize <= avail && nsize <= availtoend) { jo->fifo.windex += nsize - osize; + spin_unlock(&journal_fifo_lock); /* DF-2747 ext-2 */ rawp->recsize += bytes; return((char *)(rawp + 1) + truncbytes); } } + spin_unlock(&journal_fifo_lock); /* DF-2747 ext-fallthrough */ /* * It was not possible to extend the buffer. Commit the current * buffer and create a new one. We manually clear the BEGIN mark that @@ -682,6 +725,7 @@ int osize; rawp = *rawpp; + spin_lock(&journal_fifo_lock); /* DF-2747 */ osize = (rawp->recsize + 15) & ~15; if ((rawp->streamid & JREC_STREAMCTL_BEGIN) && @@ -689,8 +733,10 @@ (char *)rawp - jo->fifo.membase + osize) { jo->fifo.windex -= osize; + spin_unlock(&journal_fifo_lock); /* DF-2747 */ *rawpp = NULL; } else { + spin_unlock(&journal_fifo_lock); /* DF-2747 */ rawp->streamid |= JREC_STREAMCTL_ABORTED; journal_commit(jo, rawpp, 0, 1); } @@ -739,6 +785,7 @@ */ if (bytes >= 0) { KKASSERT(bytes >= 0 && bytes <= rawp->recsize - sizeof(struct journal_rawrecbeg) - sizeof(struct journal_rawrecend)); + spin_lock(&journal_fifo_lock); /* DF-2747 */ osize = (rawp->recsize + 15) & ~15; rawp->recsize = bytes + sizeof(struct journal_rawrecbeg) + sizeof(struct journal_rawrecend); @@ -754,6 +801,7 @@ journal_build_pad((void *)((char *)rawp + nsize), osize - nsize, rawp->transid + 1); } + spin_unlock(&journal_fifo_lock); /* DF-2747 */ } /*