/*-
 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/dev/virtio/block/virtio_blk.c 252707 2013-07-04 17:57:26Z bryanv $
 */

/* Driver for VirtIO block devices. */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/bio.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/sglist.h>
#include <sys/sysctl.h>
#include <sys/queue.h>
#include <sys/serialize.h>
#include <sys/buf2.h>
#include <sys/rman.h>
#include <sys/disk.h>
#include <sys/devicestat.h>

#include <dev/virtual/virtio/virtio/virtio.h>
#include <dev/virtual/virtio/virtio/virtqueue.h>
#include "virtio_blk.h"

struct vtblk_request {
	struct virtio_blk_outhdr	 vbr_hdr __aligned(16);
	struct bio			*vbr_bio;
	uint8_t				 vbr_ack;

	SLIST_ENTRY(vtblk_request)	 vbr_link;
};

enum vtblk_cache_mode {
	VTBLK_CACHE_WRITETHROUGH,
	VTBLK_CACHE_WRITEBACK,
	VTBLK_CACHE_MAX
};

struct vtblk_queue {
	struct vtblk_softc	*vtblk_sc;
	struct virtqueue	*vtblk_vq;
	struct sglist		*vtblk_sglist;
	struct bio_queue_head	 vtblk_bioq;
	SLIST_HEAD(, vtblk_request)
				 vtblk_req_free;
	struct lwkt_serialize	 vtblk_slz;
};

struct vtblk_softc {
	device_t		 vtblk_dev;
	uint64_t		 vtblk_features;
	uint32_t		 vtblk_flags;
#define VTBLK_FLAG_INDIRECT	0x0001
#define VTBLK_FLAG_READONLY	0x0002
#define VTBLK_FLAG_DETACH	0x0004
#define VTBLK_FLAG_SUSPEND	0x0008
#define VTBLK_FLAG_DUMPING	0x0010
#define VTBLK_FLAG_WC_CONFIG	0x0020

	struct disk		 vtblk_disk;
	cdev_t			 cdev;
	struct devstat		 stats;

	struct vtblk_queue	 vtblk_queues[SMP_MAXCPU];
	u_int			 vtblk_nqs;
	u_int			 vtblk_nintrs;
	int			 vtblk_vqmap[SMP_MAXCPU];

	int			 vtblk_sector_size;
	int			 vtblk_max_nsegs;
	int			 vtblk_request_count;
	enum vtblk_cache_mode	 vtblk_write_cache;

	struct vtblk_request	 vtblk_dump_request;
};

static struct virtio_feature_desc vtblk_feature_desc[] = {
	{ VIRTIO_BLK_F_BARRIER,		"HostBarrier"	},
	{ VIRTIO_BLK_F_SIZE_MAX,	"MaxSegSize"	},
	{ VIRTIO_BLK_F_SEG_MAX,		"MaxNumSegs"	},
	{ VIRTIO_BLK_F_GEOMETRY,	"DiskGeometry"	},
	{ VIRTIO_BLK_F_RO,		"ReadOnly"	},
	{ VIRTIO_BLK_F_BLK_SIZE,	"BlockSize"	},
	{ VIRTIO_BLK_F_SCSI,		"SCSICmds"	},
	{ VIRTIO_BLK_F_FLUSH,		"FlushCommand"	},
	{ VIRTIO_BLK_F_TOPOLOGY,	"Topology"	},
	{ VIRTIO_BLK_F_CONFIG_WCE,	"ConfigWCE"	},
	{ VIRTIO_BLK_F_MQ,		"MultiQueue"	},
	{ VIRTIO_BLK_F_DISCARD,		"Discard"	},
	{ VIRTIO_BLK_F_WRITE_ZEROES,	"WriteZeroes"	},

	{ 0, NULL }
};

static int	vtblk_probe(device_t);
static int	vtblk_attach(device_t);
static int	vtblk_detach(device_t);
static int	vtblk_suspend(device_t);
static int	vtblk_resume(device_t);
static int	vtblk_shutdown(device_t);

static void	vtblk_negotiate_features(struct vtblk_softc *);
static int	vtblk_alloc_intrs(struct vtblk_softc *);
static int	vtblk_maximum_segments(struct vtblk_softc *,
		    struct virtio_blk_config *);
static int	vtblk_alloc_virtqueues(struct vtblk_softc *);
static void	vtblk_set_write_cache(struct vtblk_softc *, int);
static int	vtblk_write_cache_enabled(struct vtblk_softc *,
		    struct virtio_blk_config *);
static int	vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
static void	vtblk_alloc_disk(struct vtblk_softc *,
		    struct virtio_blk_config *);
/*
 * Interface to the device switch.
 */
static d_open_t		vtblk_open;
static d_strategy_t	vtblk_strategy;
static d_dump_t		vtblk_dump;

static struct dev_ops vbd_disk_ops = {
	{ "vbd", 200, D_DISK | D_MPSAFE | D_KVABIO },
	.d_open		= vtblk_open,
	.d_close	= nullclose,
	.d_read		= physread,
	.d_write	= physwrite,
	.d_strategy	= vtblk_strategy,
	.d_dump		= vtblk_dump,
};

static void	vtblk_vq_startio(struct vtblk_queue *);
static struct vtblk_request * vtblk_bio_request(struct vtblk_queue *);
static int	vtblk_execute_request(struct vtblk_queue *,
		    struct vtblk_request *);
static void	vtblk_vq_intr(void *);

static void	vtblk_prepare_dump(struct vtblk_softc *);
static int	vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
static int	vtblk_flush_dump(struct vtblk_softc *);
static int	vtblk_poll_request(struct vtblk_softc *,
		    struct vtblk_request *);

static void	vtblk_drain_vq(struct vtblk_queue *, int);
static void	vtblk_drain(struct vtblk_softc *);

static int	vtblk_alloc_requests(struct vtblk_queue *);
static void	vtblk_free_requests(struct vtblk_softc *);
static struct vtblk_request * vtblk_dequeue_request(struct vtblk_queue *);
static void	vtblk_enqueue_request(struct vtblk_queue *,
		    struct vtblk_request *);

static int	vtblk_request_error(struct vtblk_request *);
static void	vtblk_finish_bio(struct bio *, int);

static void	vtblk_setup_sysctl(struct vtblk_softc *);
static int	vtblk_tunable_int(struct vtblk_softc *, const char *, int);

/* Tunables. */
static int vtblk_writecache_mode = -1;
TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);
static int vtblk_max_queues = SMP_MAXCPU;
TUNABLE_INT("hw.vtblk.max_queues", &vtblk_max_queues);

/* Features desired/implemented by this driver. */
#define VTBLK_FEATURES \
    (VIRTIO_BLK_F_SIZE_MAX		| \
     VIRTIO_BLK_F_SEG_MAX		| \
     VIRTIO_BLK_F_GEOMETRY		| \
     VIRTIO_BLK_F_RO			| \
     VIRTIO_BLK_F_BLK_SIZE		| \
     VIRTIO_BLK_F_FLUSH			| \
     VIRTIO_BLK_F_CONFIG_WCE		| \
     VIRTIO_BLK_F_MQ			| \
     VIRTIO_RING_F_INDIRECT_DESC)

/*
 * Each block request uses at least two segments - one for the header
 * and one for the status.
 */
#define VTBLK_MIN_SEGMENTS	2

static device_method_t vtblk_methods[] = {
	/* Device methods. */
	DEVMETHOD(device_probe,		vtblk_probe),
	DEVMETHOD(device_attach,	vtblk_attach),
	DEVMETHOD(device_detach,	vtblk_detach),
	DEVMETHOD(device_suspend,	vtblk_suspend),
	DEVMETHOD(device_resume,	vtblk_resume),
	DEVMETHOD(device_shutdown,	vtblk_shutdown),

	DEVMETHOD_END
};

static driver_t vtblk_driver = {
	"vtblk",
	vtblk_methods,
	sizeof(struct vtblk_softc)
};
static devclass_t vtblk_devclass;

DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass, NULL, NULL);
MODULE_VERSION(virtio_blk, 1);
MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);

static int
vtblk_probe(device_t dev)
{

	if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK)
		return (ENXIO);

	device_set_desc(dev, "VirtIO Block Adapter");

	return (BUS_PROBE_DEFAULT);
}

static int
vtblk_attach(device_t dev)
{
	struct vtblk_softc *sc;
	struct virtio_blk_config blkcfg;
	int error;
	int i;

	sc = device_get_softc(dev);
	sc->vtblk_dev = dev;

	virtio_set_feature_desc(dev, vtblk_feature_desc);
	vtblk_negotiate_features(sc);

	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
		sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;

	/* Get local copy of config. */
	virtio_read_device_config(dev, 0, &blkcfg,
				  sizeof(struct virtio_blk_config));

	/*
	 * With the current sglist(9) implementation, it is not easy
	 * for us to support a maximum segment size as adjacent
	 * segments are coalesced. For now, just make sure it's larger
	 * than the maximum supported transfer size.
	 */
	if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
		if (blkcfg.size_max < MAXPHYS) {
			error = ENOTSUP;
			device_printf(dev, "host requires unsupported "
			    "maximum segment size feature\n");
			return error;
		}
	}

	sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
	if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
		error = EINVAL;
		device_printf(dev, "fewer than minimum number of segments "
		    "allowed: %d\n", sc->vtblk_max_nsegs);
		return error;
	}

	if (virtio_with_feature(dev, VIRTIO_BLK_F_MQ)) {
		int max_queues =
		    vtblk_tunable_int(sc, "max_queues", vtblk_max_queues);
		sc->vtblk_nqs = min(blkcfg.num_queues, ncpus);
		sc->vtblk_nqs = min(sc->vtblk_nqs, VIRTIO_MAX_VIRTQUEUES);
		sc->vtblk_nqs = min(sc->vtblk_nqs, max_queues);
		sc->vtblk_nintrs =
		    min(virtio_intr_count(sc->vtblk_dev), sc->vtblk_nqs);
		// Limit to a 1:1 mapping of IRQs to Virtqueue for now.
		sc->vtblk_nqs = min(sc->vtblk_nqs, sc->vtblk_nintrs);
	} else {
		sc->vtblk_nqs = 1;
		sc->vtblk_nintrs = 1;
	}

	for (i = 0; i < sc->vtblk_nqs; i++) {
		struct vtblk_queue *vq = &sc->vtblk_queues[i];

		vq->vtblk_sc = sc;
		lwkt_serialize_init(&vq->vtblk_slz);
		bioq_init(&vq->vtblk_bioq);
		SLIST_INIT(&vq->vtblk_req_free);
		/*
		 * Allocate working sglist. The number of segments may be too
		 * large to safely store on the stack.
		 */
		vq->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_INTWAIT);
		if (vq->vtblk_sglist == NULL) {
			error = ENOMEM;
			device_printf(dev, "cannot allocate sglist\n");
			goto fail;
		}
	}

	for (i = 0; i < ncpus; i++) {
		// Could be improved to take the CPU topology into account.
		sc->vtblk_vqmap[i] = i % sc->vtblk_nqs;
	}

	error = vtblk_alloc_intrs(sc);
	if (error) {
		device_printf(dev, "cannot allocate interrupts\n");
		goto fail;
	}

	error = vtblk_alloc_virtqueues(sc);
	if (error) {
		device_printf(dev, "cannot allocate virtqueues\n");
		goto fail;
	}

	for (i = 0; i < sc->vtblk_nqs; i++) {
		error = virtio_bind_intr(sc->vtblk_dev, i, i,
					 vtblk_vq_intr, &sc->vtblk_queues[i]);
		if (error) {
			device_printf(dev,
			    "cannot assign virtqueue to interrupt\n");
			goto fail;
		}
	}

	for (i = 0; i < sc->vtblk_nqs; i++) {
		error = vtblk_alloc_requests(&sc->vtblk_queues[i]);
		if (error) {
			device_printf(dev, "cannot preallocate requests\n");
			goto fail;
		}
	}

	for (i = 0; i < sc->vtblk_nintrs; i++) {
		error = virtio_setup_intr(dev, i,
					  &sc->vtblk_queues[i].vtblk_slz);
		if (error) {
			device_printf(dev,
				      "cannot setup virtqueue interrupt\n");
			goto fail;
		}
	}

	for (i = 0; i < sc->vtblk_nqs; i++)
		virtqueue_enable_intr(sc->vtblk_queues[i].vtblk_vq);

	vtblk_alloc_disk(sc, &blkcfg);
	vtblk_setup_sysctl(sc);

fail:
	if (error)
		vtblk_detach(dev);

	return (error);
}

static int
vtblk_detach(device_t dev)
{
	struct vtblk_softc *sc;
	int i;

	sc = device_get_softc(dev);

	for (i = 0; i < sc->vtblk_nintrs; i++)
		virtio_teardown_intr(dev, i);

	sc->vtblk_flags |= VTBLK_FLAG_DETACH;
	// Once VTBLK_FLAG_DETACH is set, we just need to take the virtqueue
	// serializers once, to make sure that any pending d_strategy call is
	// finished, or will return ENXIO.
	for (i = 0; i < sc->vtblk_nqs; i++) {
		struct vtblk_queue *vq = &sc->vtblk_queues[i];

		lwkt_serialize_enter(&vq->vtblk_slz);
		lwkt_serialize_exit(&vq->vtblk_slz);
	}
	virtio_stop(sc->vtblk_dev);

	// Now the device is fully stopped, and we can clean up everything
	// safely.
	vtblk_drain(sc);
	if (sc->cdev != NULL) {
		disk_destroy(&sc->vtblk_disk);
		sc->cdev = NULL;
	}

	for (i = 0; i < sc->vtblk_nqs; i++) {
		struct vtblk_queue *vq = &sc->vtblk_queues[i];

		if (vq->vtblk_sglist != NULL) {
			sglist_free(vq->vtblk_sglist);
			vq->vtblk_sglist = NULL;
		}
	}

	return (0);
}

static int
vtblk_suspend(device_t dev)
{
	struct vtblk_softc *sc = device_get_softc(dev);
	int i;

	sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
	for (i = 0; i < sc->vtblk_nqs; i++) {
		lwkt_serialize_enter(&sc->vtblk_queues[i].vtblk_slz);
		/* XXX BMV: virtio_stop(), etc needed here? */
		lwkt_serialize_exit(&sc->vtblk_queues[i].vtblk_slz);
	}

	return (0);
}

static int
vtblk_resume(device_t dev)
{
	struct vtblk_softc *sc = device_get_softc(dev);
	int i;

	sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
	for (i = 0; i < sc->vtblk_nqs; i++) {
		lwkt_serialize_enter(&sc->vtblk_queues[i].vtblk_slz);
		/* XXX BMV: virtio_reinit(), etc needed here? */
#if 0 /* XXX Resume IO? */
		vtblk_vq_startio(&sc->vtblk_queues[i]);
#endif
		lwkt_serialize_exit(&sc->vtblk_queues[i].vtblk_slz);
	}

	return (0);
}

static int
vtblk_shutdown(device_t dev)
{

	return (0);
}

static int
vtblk_open(struct dev_open_args *ap)
{
	struct vtblk_softc *sc;
	cdev_t dev = ap->a_head.a_dev;
	sc = dev->si_drv1;
	if (sc == NULL)
		return (ENXIO);

	if ((ap->a_oflags & FWRITE) && (sc->vtblk_flags & VTBLK_FLAG_READONLY))
		return (EACCES);

	return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
}

static int
vtblk_dump(struct dev_dump_args *ap)
{
	struct vtblk_softc *sc;
	cdev_t dev = ap->a_head.a_dev;
        uint64_t buf_start, buf_len;
        int error;

	sc = dev->si_drv1;
	if (sc == NULL)
		return (ENXIO);

        buf_start = ap->a_offset;
        buf_len = ap->a_length;

//	lwkt_serialize_enter(&sc->vtblk_queues[0].vtblk_slz);

	if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
		vtblk_prepare_dump(sc);
		sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
	}

	if (buf_len > 0)
		error = vtblk_write_dump(sc, ap->a_virtual, buf_start,
		    buf_len);
	else if (buf_len == 0)
		error = vtblk_flush_dump(sc);
	else {
		error = EINVAL;
		sc->vtblk_flags &= ~VTBLK_FLAG_DUMPING;
	}

//	lwkt_serialize_exit(&sc->vtblk_queues[0].vtblk_slz);

	return (error);
}

/*
 * WARNING! We are using the KVABIO API and must not access memory
 *          through bp->b_data without first calling bkvasync(bp).
 */
static int
vtblk_strategy(struct dev_strategy_args *ap)
{
	struct vtblk_softc *sc;
	cdev_t dev = ap->a_head.a_dev;
	sc = dev->si_drv1;
	struct bio *bio = ap->a_bio;
	struct vtblk_queue *q = &sc->vtblk_queues[sc->vtblk_vqmap[mycpuid]];

	if (sc == NULL) {
		vtblk_finish_bio(bio, EINVAL);
		return EINVAL;
	}

	lwkt_serialize_enter(&q->vtblk_slz);
	if ((sc->vtblk_flags & VTBLK_FLAG_DETACH) == 0) {
		bioqdisksort(&q->vtblk_bioq, bio);
		vtblk_vq_startio(q);
		lwkt_serialize_exit(&q->vtblk_slz);
	} else {
		lwkt_serialize_exit(&q->vtblk_slz);
		vtblk_finish_bio(bio, ENXIO);
	}
	return 0;
}

static void
vtblk_negotiate_features(struct vtblk_softc *sc)
{
	device_t dev;
	uint64_t features;

	dev = sc->vtblk_dev;
	features = VTBLK_FEATURES;

	sc->vtblk_features = virtio_negotiate_features(dev, features);
}

/*
 * Calculate the maximum number of DMA segment supported.  Note
 * that the in/out header is encoded in the segment list.  We
 * assume that VTBLK_MIN_SEGMENTS covers that part of it so
 * we add it into the desired total.  If the SEG_MAX feature
 * is not specified we have to just assume that the host can
 * handle the maximum number of segments required for a MAXPHYS
 * sized request.
 *
 * The additional + 1 is in case a MAXPHYS-sized buffer crosses
 * a page boundary.
 */
static int
vtblk_maximum_segments(struct vtblk_softc *sc,
    struct virtio_blk_config *blkcfg)
{
	device_t dev;
	int nsegs;

	dev = sc->vtblk_dev;
	nsegs = VTBLK_MIN_SEGMENTS;

	if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
		nsegs = MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1 + nsegs);
	} else {
		nsegs = MAXPHYS / PAGE_SIZE + 1 + nsegs;
	}
	if (sc->vtblk_flags & VTBLK_FLAG_INDIRECT)
		nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);

	return (nsegs);
}

static int
vtblk_alloc_intrs(struct vtblk_softc *sc)
{
	int cnt = sc->vtblk_nintrs;
	int *cpus = NULL;
	int error;

	if (cnt > 1) {
		// We can re-use the cpuid-to-virtqueue mapping here.
		// TODO: Actually take CPU topology into account, for a better
		//       distribution.
		cpus = sc->vtblk_vqmap;
	}

	error = virtio_intr_alloc(sc->vtblk_dev, &cnt, 0, cpus);
	if (error != 0)
		return (error);
	else if (cnt != sc->vtblk_nqs)
		return (ENXIO);

	return (0);
}

static int
vtblk_alloc_virtqueues(struct vtblk_softc *sc)
{
	device_t dev = sc->vtblk_dev;
	struct vq_alloc_info *vq_info =
	    kmalloc(sc->vtblk_nqs * sizeof(struct vq_alloc_info),
	    M_TEMP, M_WAITOK | M_ZERO);
	int i;
	int error;

	for (i = 0; i < sc->vtblk_nqs; i++) {
		VQ_ALLOC_INFO_INIT(&vq_info[i], sc->vtblk_max_nsegs,
		    &sc->vtblk_queues[i].vtblk_vq, "%s request %d",
		    device_get_nameunit(dev), i);
	}

	error = virtio_alloc_virtqueues(dev, sc->vtblk_nqs, vq_info);
	kfree(vq_info, M_TEMP);
	return error;
}

static void
vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
{

	/* Set either writeback (1) or writethrough (0) mode. */
	virtio_write_dev_config_1(sc->vtblk_dev,
	    offsetof(struct virtio_blk_config, writeback), wc);
}

static int
vtblk_write_cache_enabled(struct vtblk_softc *sc,
    struct virtio_blk_config *blkcfg)
{
	int wc;

	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
		wc = vtblk_tunable_int(sc, "writecache_mode",
		    vtblk_writecache_mode);
		if (wc >= 0 && wc < VTBLK_CACHE_MAX)
			vtblk_set_write_cache(sc, wc);
		else
			wc = blkcfg->writeback;
	} else
		wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_FLUSH);

	return (wc);
}

static int
vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
{
	struct vtblk_softc *sc;
	int oldwc, wc, error;

	sc = oidp->oid_arg1;
	oldwc = wc = sc->vtblk_write_cache;

	error = sysctl_handle_int(oidp, &wc, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
		return (EPERM);
	if (wc < 0 || wc >= VTBLK_CACHE_MAX)
		return (EINVAL);

	if (oldwc == wc)
		return (0);

	// Not used outside this SYSCTL right now, so no locking needed.
	//lwkt_serialize_enter(&sc->vtblk_queues[0].vtblk_slz);
	sc->vtblk_write_cache = wc;
	vtblk_set_write_cache(sc, sc->vtblk_write_cache);
	//lwkt_serialize_exit(&sc->vtblk_queues[0].vtblk_slz);

	return (0);
}

static void
vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
{
	struct disk_info info;

	/* construct the disk_info */
	bzero(&info, sizeof(info));

	if (virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_BLK_SIZE))
		sc->vtblk_sector_size = blkcfg->blk_size;
	else
		sc->vtblk_sector_size = 512;

	/* blkcfg->capacity is always expressed in 512 byte sectors. */
	info.d_media_blksize = sc->vtblk_sector_size;
	info.d_media_blocks = blkcfg->capacity * 512 / info.d_media_blksize;

	if (virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_GEOMETRY)) {
		info.d_ncylinders = blkcfg->geometry.cylinders;
		info.d_nheads = blkcfg->geometry.heads;
		info.d_secpertrack = blkcfg->geometry.sectors;
		info.d_secpercyl = info.d_secpertrack * info.d_nheads;

		/*
		 * If the virtio device is reporting a legacy cylinder count,
		 * recalculate ncylinders based on the media size
		 */
		if (info.d_ncylinders == 16383) {
			info.d_ncylinders = info.d_media_blocks /
			    (info.d_nheads * info.d_secpertrack);
			device_printf(sc->vtblk_dev,
				      "Virtio: ncylinders at legacy maximum "
				      "(16383), recalculating to %d\n",
				      info.d_ncylinders);
		}
	} else {
		/* Fabricate a geometry */
		info.d_secpertrack = 1024;
		info.d_nheads = 1;
		info.d_secpercyl = info.d_secpertrack * info.d_nheads;
		info.d_ncylinders =
		    (u_int)(info.d_media_blocks / info.d_secpercyl);
	}

	if (vtblk_write_cache_enabled(sc, blkcfg) != 0)
		sc->vtblk_write_cache = VTBLK_CACHE_WRITEBACK;
	else
		sc->vtblk_write_cache = VTBLK_CACHE_WRITETHROUGH;

	devstat_add_entry(&sc->stats, "vbd", device_get_unit(sc->vtblk_dev),
			  DEV_BSIZE, DEVSTAT_ALL_SUPPORTED,
			  DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER,
			  DEVSTAT_PRIORITY_DISK);

	/* attach a generic disk device to ourselves */
	sc->cdev = disk_create(device_get_unit(sc->vtblk_dev), &sc->vtblk_disk,
			       &vbd_disk_ops);

	sc->cdev->si_drv1 = sc;
	sc->cdev->si_iosize_max = MAXPHYS;
	disk_setdiskinfo(&sc->vtblk_disk, &info);
	if (virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_BLK_SIZE)) {
		device_printf(sc->vtblk_dev, "Block size: %u\n",
		    sc->vtblk_sector_size);
	}
	device_printf(sc->vtblk_dev,
	    "%juMB (%ju %d byte sectors: %dH %dS/T %dC)\n",
	    ((uintmax_t)info.d_media_blocks * info.d_media_blksize) >> 20,
	    (uintmax_t)info.d_media_blocks, info.d_media_blksize,
	    blkcfg->geometry.heads, blkcfg->geometry.sectors,
	    blkcfg->geometry.cylinders);
}

static void
vtblk_vq_startio(struct vtblk_queue *q)
{
	struct vtblk_softc *sc = q->vtblk_sc;
	struct virtqueue *vq;
	struct vtblk_request *req;
	int enq;

	vq = q->vtblk_vq;
	enq = 0;

	ASSERT_SERIALIZED(&q->vtblk_slz);

	if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
		return;

	while (!virtqueue_full(vq)) {
		req = vtblk_bio_request(q);
		if (req == NULL)
			break;

		if (vtblk_execute_request(q, req) != 0) {
			bioqdisksort(&q->vtblk_bioq, req->vbr_bio);
			vtblk_enqueue_request(q, req);
			break;
		}
		devstat_start_transaction(&sc->stats);

		enq++;
	}

	if (enq > 0)
		virtqueue_notify(vq, &q->vtblk_slz);
}

static struct vtblk_request *
vtblk_bio_request(struct vtblk_queue *q)
{
	struct bio_queue_head *bioq;
	struct vtblk_request *req;
	struct bio *bio;
	struct buf *bp;

	bioq = &q->vtblk_bioq;

	if (bioq_first(bioq) == NULL)
		return (NULL);

	req = vtblk_dequeue_request(q);
	if (req == NULL)
		return (NULL);

	bio = bioq_takefirst(bioq);
	req->vbr_bio = bio;
	req->vbr_ack = -1;
	req->vbr_hdr.ioprio = 1;
	bp = bio->bio_buf;

	switch (bp->b_cmd) {
	case BUF_CMD_FLUSH:
		req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
		break;
	case BUF_CMD_READ:
		req->vbr_hdr.type = VIRTIO_BLK_T_IN;
		req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
		break;
	case BUF_CMD_WRITE:
		req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
		req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
		break;
	default:
		KASSERT(0, ("bio with unhandled cmd: %d", bp->b_cmd));
		req->vbr_hdr.type = -1;
		break;
	}

	return (req);
}

static int
vtblk_execute_request(struct vtblk_queue *q, struct vtblk_request *req)
{
	struct sglist *sg;
	struct bio *bio;
	struct buf *bp;
	int writable, error;

	sg = q->vtblk_sglist;
	bio = req->vbr_bio;
	bp = bio->bio_buf;
	writable = 0;

	/*
	 * sglist is live throughout this subroutine.
	 */
	error = sglist_append(sg, &req->vbr_hdr,
			      sizeof(struct virtio_blk_outhdr));
	KASSERT(error == 0, ("error adding header to sglist"));
	KASSERT(sg->sg_nseg == 1,
	    ("header spanned multiple segments: %d", sg->sg_nseg));

	if (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_WRITE) {
		error = sglist_append(sg, bp->b_data, bp->b_bcount);
		KASSERT(error == 0, ("error adding buffer to sglist"));

		/* BUF_CMD_READ means the host writes into our buffer. */
		if (bp->b_cmd == BUF_CMD_READ)
			writable += sg->sg_nseg - 1;
	}

	error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
	KASSERT(error == 0, ("error adding ack to sglist"));
	writable++;

	KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
	    ("fewer than min segments: %d", sg->sg_nseg));

	error = virtqueue_enqueue(q->vtblk_vq, req, sg,
				  sg->sg_nseg - writable, writable);

	sglist_reset(sg);

	return (error);
}

static void
vtblk_vq_intr(void *arg)
{
	struct vtblk_queue *q = arg;
	struct vtblk_softc *sc = q->vtblk_sc;
	struct virtqueue *vq = q->vtblk_vq;
	struct vtblk_request *req;
	struct bio *bio;
	struct buf *bp;

	ASSERT_SERIALIZED(&q->vtblk_slz);

	if (!virtqueue_pending(vq))
		return;

	lwkt_serialize_handler_disable(&q->vtblk_slz);
	virtqueue_disable_intr(q->vtblk_vq);

retry:
	if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
		return;

	while ((req = virtqueue_dequeue(vq, NULL)) != NULL) {
		bio = req->vbr_bio;
		bp = bio->bio_buf;

		if (req->vbr_ack == VIRTIO_BLK_S_OK) {
			bp->b_resid = 0;
		} else {
			bp->b_flags |= B_ERROR;
			if (req->vbr_ack == VIRTIO_BLK_S_UNSUPP) {
				bp->b_error = ENOTSUP;
			} else {
				bp->b_error = EIO;
			}
		}

		devstat_end_transaction_buf(&sc->stats, bio->bio_buf);

		lwkt_serialize_exit(&q->vtblk_slz);
		/*
		 * Unlocking the controller around biodone() does not allow
		 * processing further device interrupts; when we queued
		 * vtblk_vq_intr, we disabled interrupts. It will allow
		 * concurrent vtblk_strategy/_startio command dispatches.
		 */
		biodone(bio);
		lwkt_serialize_enter(&q->vtblk_slz);

		vtblk_enqueue_request(q, req);
	}

	vtblk_vq_startio(q);

	if (virtqueue_enable_intr(vq) != 0) {
		/*
		 * If new virtqueue entries appeared immediately after
		 * enabling interrupts, process them now. Release and
		 * retake softcontroller lock to try to avoid blocking
		 * I/O dispatch for too long.
		 */
		virtqueue_disable_intr(vq);
		goto retry;
	}
	lwkt_serialize_handler_enable(&q->vtblk_slz);
}

static void
vtblk_prepare_dump(struct vtblk_softc *sc)
{
	device_t dev;
	int i;

	dev = sc->vtblk_dev;

	for (i = 0 ; i < sc->vtblk_nqs; i++)
		virtqueue_disable_intr(sc->vtblk_queues[i].vtblk_vq);

	virtio_stop(sc->vtblk_dev);

	/*
	 * Drain all requests caught in-flight in the virtqueues,
	 * skipping biodone(). When dumping, only one request is
	 * outstanding at a time, and we just poll the virtqueue
	 * for the response.
	 */
	for (i = 0 ; i < sc->vtblk_nqs; i++)
		vtblk_drain_vq(&sc->vtblk_queues[i], 1);

	if (virtio_reinit(dev, sc->vtblk_features) != 0) {
		panic("%s: cannot reinit VirtIO block device during dump",
		    device_get_nameunit(dev));
	}

	for (i = 0 ; i < sc->vtblk_nqs; i++)
		virtqueue_disable_intr(sc->vtblk_queues[i].vtblk_vq);
	virtio_reinit_complete(dev);
}

static int
vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset,
    size_t length)
{
	struct bio bio;
	struct buf bp;
	struct vtblk_request *req;

	req = &sc->vtblk_dump_request;
	req->vbr_ack = -1;
	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
	req->vbr_hdr.ioprio = 1;
	req->vbr_hdr.sector = offset / 512;

	req->vbr_bio = &bio;
	bzero(&bio, sizeof(struct bio));
	bzero(&bp, sizeof(struct buf));

	bio.bio_buf = &bp;
	bp.b_cmd = BUF_CMD_WRITE;
	bp.b_data = virtual;
	bp.b_bcount = length;

	return (vtblk_poll_request(sc, req));
}

static int
vtblk_flush_dump(struct vtblk_softc *sc)
{
	struct bio bio;
	struct buf bp;
	struct vtblk_request *req;

	req = &sc->vtblk_dump_request;
	req->vbr_ack = -1;
	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
	req->vbr_hdr.ioprio = 1;
	req->vbr_hdr.sector = 0;

	req->vbr_bio = &bio;
	bzero(&bio, sizeof(struct bio));
	bzero(&bp, sizeof(struct buf));

	bio.bio_buf = &bp;
	bp.b_cmd = BUF_CMD_FLUSH;

	return (vtblk_poll_request(sc, req));
}

static int
vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
{
	struct virtqueue *vq;
	int error;

	vq = sc->vtblk_queues[0].vtblk_vq;

	if (!virtqueue_empty(vq))
		return (EBUSY);

	error = vtblk_execute_request(&sc->vtblk_queues[0], req);
	if (error)
		return (error);

	virtqueue_notify(vq, NULL);
	virtqueue_poll(vq, NULL);

	error = vtblk_request_error(req);
	if (error && bootverbose) {
		device_printf(sc->vtblk_dev,
		    "%s: IO error: %d\n", __func__, error);
	}

	return (error);
}

static void
vtblk_drain_vq(struct vtblk_queue *q, int skip_done)
{
	struct virtqueue *vq = q->vtblk_vq;
	struct vtblk_request *req;
	int last;

	last = 0;

	while ((req = virtqueue_drain(vq, &last)) != NULL) {
		if (!skip_done)
			vtblk_finish_bio(req->vbr_bio, ENXIO);

		vtblk_enqueue_request(q, req);
	}

	KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
}

static void
vtblk_drain(struct vtblk_softc *sc)
{
	struct vtblk_queue *q;
	struct bio_queue_head *bioq;
	struct bio *bio;
	int i;

	for (i = 0; i < sc->vtblk_nqs; i++) {
		q = &sc->vtblk_queues[i];
		bioq = &q->vtblk_bioq;

		if (q->vtblk_vq != NULL)
			vtblk_drain_vq(q, 0);

		while (bioq_first(bioq) != NULL) {
			bio = bioq_takefirst(bioq);
			vtblk_finish_bio(bio, ENXIO);
		}
	}

	vtblk_free_requests(sc);
}

static int
vtblk_alloc_requests(struct vtblk_queue *vq)
{
	struct vtblk_softc *sc = vq->vtblk_sc;
	struct vtblk_request *req;
	int i, nreqs;

	nreqs = virtqueue_size(vq->vtblk_vq);

	/*
	 * Preallocate sufficient requests to keep the virtqueue full. Each
	 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
	 * the number allocated when indirect descriptors are not available.
	 */
	if ((sc->vtblk_flags & VTBLK_FLAG_INDIRECT) == 0)
		nreqs /= VTBLK_MIN_SEGMENTS;

	for (i = 0; i < nreqs; i++) {
		req = contigmalloc(sizeof(struct vtblk_request), M_DEVBUF,
		    M_WAITOK, 0, BUS_SPACE_MAXADDR, 16, 0);
		if (req == NULL)
			return (ENOMEM);

		KKASSERT(sglist_count(&req->vbr_hdr, sizeof(req->vbr_hdr))
		    == 1);
		KKASSERT(sglist_count(&req->vbr_ack, sizeof(req->vbr_ack))
		    == 1);

		sc->vtblk_request_count++;
		vtblk_enqueue_request(vq, req);
	}

	return (0);
}

static void
vtblk_free_requests(struct vtblk_softc *sc)
{
	struct vtblk_request *req;
	int i;

	for (i = 0; i < sc->vtblk_nqs; i++) {
		struct vtblk_queue *q = &sc->vtblk_queues[i];
		while ((req = vtblk_dequeue_request(q)) != NULL) {
			sc->vtblk_request_count--;
			contigfree(req, sizeof(struct vtblk_request), M_DEVBUF);
		}
	}

	KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
}

static struct vtblk_request *
vtblk_dequeue_request(struct vtblk_queue *q)
{
	struct vtblk_request *req;

	req = SLIST_FIRST(&q->vtblk_req_free);
	if (req != NULL)
		SLIST_REMOVE_HEAD(&q->vtblk_req_free, vbr_link);

	return (req);
}

static void
vtblk_enqueue_request(struct vtblk_queue *vq, struct vtblk_request *req)
{

	bzero(req, sizeof(struct vtblk_request));
	SLIST_INSERT_HEAD(&vq->vtblk_req_free, req, vbr_link);
}

static int
vtblk_request_error(struct vtblk_request *req)
{
	int error;

	switch (req->vbr_ack) {
	case VIRTIO_BLK_S_OK:
		error = 0;
		break;
	case VIRTIO_BLK_S_UNSUPP:
		error = ENOTSUP;
		break;
	default:
		error = EIO;
		break;
	}

	return (error);
}

static void
vtblk_finish_bio(struct bio *bio, int error)
{

	biodone(bio);
}

static void
vtblk_setup_sysctl(struct vtblk_softc *sc)
{
	device_t dev;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *tree;
	struct sysctl_oid_list *child;

	dev = sc->vtblk_dev;
	ctx = device_get_sysctl_ctx(dev);
	tree = device_get_sysctl_tree(dev);
	child = SYSCTL_CHILDREN(tree);

	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "writecache_mode",
	    CTLTYPE_INT | CTLFLAG_RW, sc, 0, vtblk_write_cache_sysctl,
	    "I", "Write cache mode (writethrough (0) or writeback (1))");
}

static int
vtblk_tunable_int(struct vtblk_softc *sc, const char *knob, int def)
{
	char path[64];

	ksnprintf(path, sizeof(path),
	    "hw.vtblk.%d.%s", device_get_unit(sc->vtblk_dev), knob);
	TUNABLE_INT_FETCH(path, &def);

	return (def);
}