/*-
 * Copyright (c) 2009-2012,2016 Microsoft Corp.
 * Copyright (c) 2012 NetApp Inc.
 * Copyright (c) 2012 Citrix Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "opt_acpi.h"

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/rman.h>
#include <sys/systimer.h>
#include <sys/thread.h>
#include <sys/thread2.h>

#include <machine/intr_machdep.h>
#include <machine/smp.h>

#include <dev/virtual/hyperv/hyperv_busdma.h>
#include <dev/virtual/hyperv/hyperv_machdep.h>
#include <dev/virtual/hyperv/hyperv_reg.h>
#include <dev/virtual/hyperv/hyperv_var.h>
#include <dev/virtual/hyperv/vmbus/vmbus_reg.h>
#include <dev/virtual/hyperv/vmbus/vmbus_var.h>

#include "acpi.h"
#include "acpi_if.h"
#include "pcib_if.h"

#define MSR_HV_STIMER0_CFG_SINT		\
	((((uint64_t)VMBUS_SINT_TIMER) << MSR_HV_STIMER_CFG_SINT_SHIFT) & \
	 MSR_HV_STIMER_CFG_SINT_MASK)

/*
 * Additionally required feature:
 * - SynIC is needed for interrupt generation.
 */
#define CPUID_HV_TIMER_MASK		(CPUID_HV_MSR_SYNIC |		\
					 CPUID_HV_MSR_SYNTIMER)

/*
 * NOTE: DO NOT CHANGE THIS.
 */
#define VMBUS_SINT_MESSAGE		2
/*
 * NOTE:
 * - DO NOT set it to the same value as VMBUS_SINT_MESSAGE.
 * - DO NOT set it to 0.
 */
#define VMBUS_SINT_TIMER		4

/*
 * NOTE: DO NOT CHANGE THESE
 */
#define VMBUS_CONNID_MESSAGE		1
#define VMBUS_CONNID_EVENT		2

struct vmbus_msghc {
	struct hypercall_postmsg_in	*mh_inprm;
	struct hypercall_postmsg_in	mh_inprm_save;
	struct hyperv_dma		mh_inprm_dma;

	struct vmbus_message		*mh_resp;
	struct vmbus_message		mh_resp0;
};

struct vmbus_msghc_ctx {
	struct vmbus_msghc		*mhc_free;
	struct lwkt_token		mhc_free_token;
	uint32_t			mhc_flags;

	struct vmbus_msghc		*mhc_active;
	struct lwkt_token		mhc_active_token;
};

#define VMBUS_MSGHC_CTXF_DESTROY	0x0001

static int			vmbus_probe(device_t);
static int			vmbus_attach(device_t);
static int			vmbus_detach(device_t);
static void			vmbus_intr(void *);
static void			vmbus_timer_intr_reload(struct cputimer_intr *,
				    sysclock_t);
static void			vmbus_timer_intr_pcpuhand(
				    struct cputimer_intr *);
static void			vmbus_timer_intr_restart(
				    struct cputimer_intr *);

static int			vmbus_dma_alloc(struct vmbus_softc *);
static void			vmbus_dma_free(struct vmbus_softc *);
static int			vmbus_intr_setup(struct vmbus_softc *);
static void			vmbus_intr_teardown(struct vmbus_softc *);
static void			vmbus_synic_setup(void *);
static void			vmbus_synic_teardown(void *);
static void			vmbus_timer_stop(void *);
static void			vmbus_timer_config(void *);
static int			vmbus_init(struct vmbus_softc *);
static int			vmbus_init_contact(struct vmbus_softc *,
				    uint32_t);
static void			vmbus_timer_restart(void *);
static void			vmbus_timer_msgintr(struct vmbus_pcpu_data *);

static void			vmbus_chan_msgproc(struct vmbus_softc *,
				    const struct vmbus_message *);

static struct vmbus_msghc_ctx	*vmbus_msghc_ctx_create(bus_dma_tag_t);
static void			vmbus_msghc_ctx_destroy(
				    struct vmbus_msghc_ctx *);
static void			vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *);
static struct vmbus_msghc	*vmbus_msghc_alloc(bus_dma_tag_t);
static void			vmbus_msghc_free(struct vmbus_msghc *);
static struct vmbus_msghc	*vmbus_msghc_get1(struct vmbus_msghc_ctx *,
				    uint32_t);

static device_method_t vmbus_methods[] = {
	/* Device interface */
	DEVMETHOD(device_probe,			vmbus_probe),
	DEVMETHOD(device_attach,		vmbus_attach),
	DEVMETHOD(device_detach,		vmbus_detach),
	DEVMETHOD(device_shutdown,		bus_generic_shutdown),
	DEVMETHOD(device_suspend,		bus_generic_suspend),
	DEVMETHOD(device_resume,		bus_generic_resume),

	DEVMETHOD_END
};

static driver_t vmbus_driver = {
	"vmbus",
	vmbus_methods,
	sizeof(struct vmbus_softc)
};

static devclass_t vmbus_devclass;

DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, NULL, NULL);
MODULE_DEPEND(vmbus, acpi, 1, 1, 1);
MODULE_VERSION(vmbus, 1);

static struct cputimer_intr vmbus_cputimer_intr = {
	.freq = HYPERV_TIMER_FREQ,
	.reload = vmbus_timer_intr_reload,
	.enable = cputimer_intr_default_enable,
	.config = cputimer_intr_default_config,
	.restart = vmbus_timer_intr_restart,
	.pmfixup = cputimer_intr_default_pmfixup,
	.initclock = cputimer_intr_default_initclock,
	.pcpuhand = vmbus_timer_intr_pcpuhand,
	.next = SLIST_ENTRY_INITIALIZER,
	.name = "hyperv",
	.type = CPUTIMER_INTR_VMM,
	.prio = CPUTIMER_INTR_PRIO_VMM,
	.caps = CPUTIMER_INTR_CAP_PS,
	.priv = NULL
};

static const uint32_t	vmbus_version[] = {
	VMBUS_VERSION_WIN8_1,
	VMBUS_VERSION_WIN8,
	VMBUS_VERSION_WIN7,
	VMBUS_VERSION_WS2008
};

static int		vmbus_timer_intr_enable = 1;
TUNABLE_INT("hw.vmbus.timer_intr.enable", &vmbus_timer_intr_enable);

static int
vmbus_probe(device_t dev)
{
	char *id[] = { "VMBUS", NULL };

	if (ACPI_ID_PROBE(device_get_parent(dev), dev, id) == NULL ||
	    device_get_unit(dev) != 0 || vmm_guest != VMM_GUEST_HYPERV ||
	    (hyperv_features & CPUID_HV_MSR_SYNIC) == 0)
		return (ENXIO);

	device_set_desc(dev, "Hyper-V vmbus");

	return (0);
}

static int
vmbus_attach(device_t dev)
{
	struct vmbus_softc *sc = device_get_softc(dev);
	int error, cpu, use_timer;

	/*
	 * Basic setup.
	 */
	sc->vmbus_dev = dev;
	for (cpu = 0; cpu < ncpus; ++cpu) {
		struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);

		psc->sc = sc;
		psc->cpuid = cpu;
		psc->timer_last = UINT64_MAX;
	}

	/*
	 * Should we use interrupt timer?
	 */
	use_timer = 0;
	if (device_get_unit(dev) == 0 &&
	    (hyperv_features & CPUID_HV_TIMER_MASK) == CPUID_HV_TIMER_MASK &&
	    hyperv_tc64 != NULL)
		use_timer = 1;

	/*
	 * Create context for "post message" Hypercalls
	 */
	sc->vmbus_msg_hc = vmbus_msghc_ctx_create(
	    bus_get_dma_tag(sc->vmbus_dev));
	if (sc->vmbus_msg_hc == NULL)
		return ENXIO;

	/*
	 * Allocate DMA stuffs.
	 */
	error = vmbus_dma_alloc(sc);
	if (error)
		goto failed;

	/*
	 * Setup interrupt.
	 */
	error = vmbus_intr_setup(sc);
	if (error)
		goto failed;

	if (use_timer) {
		/*
		 * Make sure that interrupt timer is stopped.
		 */
		lwkt_cpusync_simple(smp_active_mask, vmbus_timer_stop, sc);
	}

	/*
	 * Setup SynIC.
	 */
	lwkt_cpusync_simple(smp_active_mask, vmbus_synic_setup, sc);
	sc->vmbus_flags |= VMBUS_FLAG_SYNIC;

	/*
	 * Initialize vmbus.
	 */
	error = vmbus_init(sc);
	if (error)
		goto failed;

	if (use_timer) {
		/*
		 * Configure and register vmbus interrupt timer.
		 */
		lwkt_cpusync_simple(smp_active_mask, vmbus_timer_config, sc);
		vmbus_cputimer_intr.priv = sc;
		cputimer_intr_register(&vmbus_cputimer_intr);
		if (vmbus_timer_intr_enable)
			cputimer_intr_select(&vmbus_cputimer_intr, 0);
	}

	return 0;
failed:
	vmbus_detach(dev);
	return error;
}

static int
vmbus_detach(device_t dev)
{
	struct vmbus_softc *sc = device_get_softc(dev);

	/* TODO: uninitialize vmbus. */
	/* TODO: stop and deregister timer */

	if (sc->vmbus_flags & VMBUS_FLAG_SYNIC)
		lwkt_cpusync_simple(smp_active_mask, vmbus_synic_teardown, sc);
	vmbus_intr_teardown(sc);
	vmbus_dma_free(sc);

	if (sc->vmbus_msg_hc != NULL) {
		vmbus_msghc_ctx_destroy(sc->vmbus_msg_hc);
		sc->vmbus_msg_hc = NULL;
	}
	return (0);
}

static __inline void
vmbus_msg_reset(volatile struct vmbus_message *msg)
{
	msg->msg_type = HYPERV_MSGTYPE_NONE;
	/*
	 * Make sure that the write to msg_type (i.e. set to
	 * HYPERV_MSGTYPE_NONE) happens before we read the
	 * msg_flags and send EOM to the hypervisor.
	 */
	cpu_mfence();
	if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) {
		/*
		 * Ask the hypervisor to rescan message queue,
		 * and deliver new message if any.
		 */
		wrmsr(MSR_HV_EOM, 0);
	}
}

static void
vmbus_intr(void *xpsc)
{
	struct vmbus_pcpu_data *psc = xpsc;
	volatile struct vmbus_message *msg;

	msg = psc->message + VMBUS_SINT_MESSAGE;
	while (__predict_false(msg->msg_type != HYPERV_MSGTYPE_NONE)) {
		if (msg->msg_type == HYPERV_MSGTYPE_CHANNEL) {
			/* Channel message */
			vmbus_chan_msgproc(psc->sc,
			    __DEVOLATILE(const struct vmbus_message *, msg));
		}
		vmbus_msg_reset(msg);
	}
}

static __inline void
vmbus_timer_oneshot(struct vmbus_pcpu_data *psc, uint64_t current)
{
	psc->timer_last = current;
	wrmsr(MSR_HV_STIMER0_COUNT, current);
}

static void
vmbus_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload)
{
	struct globaldata *gd = mycpu;
	struct vmbus_softc *sc = cti->priv;
	struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, gd->gd_cpuid);
	uint64_t current;

	if ((ssysclock_t)reload < 0)		/* neg value */
		reload = 1;
	reload = muldivu64(reload, cti->freq, sys_cputimer->freq);
	current = hyperv_tc64() + reload;

	if (gd->gd_timer_running) {
		if (current < psc->timer_last)
			vmbus_timer_oneshot(psc, current);
	} else {
		gd->gd_timer_running = 1;
		vmbus_timer_oneshot(psc, current);
	}
}

static void
vmbus_timer_intr_pcpuhand(struct cputimer_intr *cti)
{
	struct vmbus_softc *sc = cti->priv;
	struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid);

	vmbus_timer_msgintr(psc);
}

static void
vmbus_timer_intr_restart(struct cputimer_intr *cti)
{
	lwkt_send_ipiq_mask(smp_active_mask, vmbus_timer_restart, cti->priv);
}

static struct vmbus_msghc *
vmbus_msghc_alloc(bus_dma_tag_t parent_dtag)
{
	struct vmbus_msghc *mh;

	mh = kmalloc(sizeof(*mh), M_DEVBUF, M_WAITOK | M_ZERO);

	mh->mh_inprm = hyperv_dmamem_alloc(parent_dtag,
	    HYPERCALL_POSTMSGIN_ALIGN, 0, HYPERCALL_POSTMSGIN_SIZE,
	    &mh->mh_inprm_dma, BUS_DMA_WAITOK);
	if (mh->mh_inprm == NULL) {
		kfree(mh, M_DEVBUF);
		return NULL;
	}
	return mh;
}

static void
vmbus_msghc_free(struct vmbus_msghc *mh)
{
	hyperv_dmamem_free(&mh->mh_inprm_dma, mh->mh_inprm);
	kfree(mh, M_DEVBUF);
}

static void
vmbus_msghc_ctx_free(struct vmbus_msghc_ctx *mhc)
{
	KASSERT(mhc->mhc_active == NULL, ("still have active msg hypercall"));
	KASSERT(mhc->mhc_free == NULL, ("still have hypercall msg"));

	lwkt_token_uninit(&mhc->mhc_free_token);
	lwkt_token_uninit(&mhc->mhc_active_token);
	kfree(mhc, M_DEVBUF);
}

static struct vmbus_msghc_ctx *
vmbus_msghc_ctx_create(bus_dma_tag_t parent_dtag)
{
	struct vmbus_msghc_ctx *mhc;

	mhc = kmalloc(sizeof(*mhc), M_DEVBUF, M_WAITOK | M_ZERO);
	lwkt_token_init(&mhc->mhc_free_token, "msghcf");
	lwkt_token_init(&mhc->mhc_active_token, "msghca");

	mhc->mhc_free = vmbus_msghc_alloc(parent_dtag);
	if (mhc->mhc_free == NULL) {
		vmbus_msghc_ctx_free(mhc);
		return NULL;
	}
	return mhc;
}

static struct vmbus_msghc *
vmbus_msghc_get1(struct vmbus_msghc_ctx *mhc, uint32_t dtor_flag)
{
	struct vmbus_msghc *mh;

	lwkt_gettoken(&mhc->mhc_free_token);

	while ((mhc->mhc_flags & dtor_flag) == 0 && mhc->mhc_free == NULL)
		tsleep(&mhc->mhc_free, 0, "gmsghc", 0);
	if (mhc->mhc_flags & dtor_flag) {
		/* Being destroyed */
		mh = NULL;
	} else {
		mh = mhc->mhc_free;
		KASSERT(mh != NULL, ("no free hypercall msg"));
		KASSERT(mh->mh_resp == NULL,
		    ("hypercall msg has pending response"));
		mhc->mhc_free = NULL;
	}

	lwkt_reltoken(&mhc->mhc_free_token);

	return mh;
}

struct vmbus_msghc *
vmbus_msghc_get(struct vmbus_softc *sc, size_t dsize)
{
	struct hypercall_postmsg_in *inprm;
	struct vmbus_msghc *mh;

	if (dsize > HYPERCALL_POSTMSGIN_DSIZE_MAX)
		return NULL;

	mh = vmbus_msghc_get1(sc->vmbus_msg_hc, VMBUS_MSGHC_CTXF_DESTROY);
	if (mh == NULL)
		return NULL;

	inprm = mh->mh_inprm;
	memset(inprm, 0, HYPERCALL_POSTMSGIN_SIZE);
	inprm->hc_connid = VMBUS_CONNID_MESSAGE;
	inprm->hc_msgtype = HYPERV_MSGTYPE_CHANNEL;
	inprm->hc_dsize = dsize;

	return mh;
}

void
vmbus_msghc_put(struct vmbus_softc *sc, struct vmbus_msghc *mh)
{
	struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;

	KASSERT(mhc->mhc_active == NULL, ("msg hypercall is active"));
	mh->mh_resp = NULL;

	lwkt_gettoken(&mhc->mhc_free_token);
	KASSERT(mhc->mhc_free == NULL, ("has free hypercall msg"));
	mhc->mhc_free = mh;
	lwkt_reltoken(&mhc->mhc_free_token);
	wakeup(&mhc->mhc_free);
}

void *
vmbus_msghc_dataptr(struct vmbus_msghc *mh)
{
	return mh->mh_inprm->hc_data;
}

static void
vmbus_msghc_ctx_destroy(struct vmbus_msghc_ctx *mhc)
{
	struct vmbus_msghc *mh;

	lwkt_gettoken(&mhc->mhc_free_token);
	mhc->mhc_flags |= VMBUS_MSGHC_CTXF_DESTROY;
	lwkt_reltoken(&mhc->mhc_free_token);
	wakeup(&mhc->mhc_free);

	mh = vmbus_msghc_get1(mhc, 0);
	if (mh == NULL)
		panic("can't get msghc");

	vmbus_msghc_free(mh);
	vmbus_msghc_ctx_free(mhc);
}

int
vmbus_msghc_exec_noresult(struct vmbus_msghc *mh)
{
	int i, wait_ticks = 1;

	/*
	 * Save the input parameter so that we could restore the input
	 * parameter if the Hypercall failed.
	 *
	 * XXX
	 * Is this really necessary?!  i.e. Will the Hypercall ever
	 * overwrite the input parameter?
	 */
	memcpy(&mh->mh_inprm_save, mh->mh_inprm, HYPERCALL_POSTMSGIN_SIZE);

	/*
	 * In order to cope with transient failures, e.g. insufficient
	 * resources on host side, we retry the post message Hypercall
	 * several times.  20 retries seem sufficient.
	 */
#define HC_RETRY_MAX	20

	for (i = 0; i < HC_RETRY_MAX; ++i) {
		uint64_t status;

		status = hypercall_post_message(mh->mh_inprm_dma.hv_paddr);
		if (status == HYPERCALL_STATUS_SUCCESS)
			return 0;

		tsleep(&status, 0, "hcpmsg", wait_ticks);
		if (wait_ticks < hz)
			wait_ticks *= 2;

		/* Restore input parameter and try again */
		memcpy(mh->mh_inprm, &mh->mh_inprm_save,
		    HYPERCALL_POSTMSGIN_SIZE);
	}

#undef HC_RETRY_MAX

	return EIO;
}

int
vmbus_msghc_exec(struct vmbus_softc *sc, struct vmbus_msghc *mh)
{
	struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;
	int error;

	KASSERT(mh->mh_resp == NULL, ("hypercall msg has pending response"));

	lwkt_gettoken(&mhc->mhc_active_token);
	KASSERT(mhc->mhc_active == NULL, ("pending active msg hypercall"));
	mhc->mhc_active = mh;
	lwkt_reltoken(&mhc->mhc_active_token);

	error = vmbus_msghc_exec_noresult(mh);
	if (error) {
		lwkt_gettoken(&mhc->mhc_active_token);
		KASSERT(mhc->mhc_active == mh, ("msghc mismatch"));
		mhc->mhc_active = NULL;
		lwkt_reltoken(&mhc->mhc_active_token);
	}
	return error;
}

const struct vmbus_message *
vmbus_msghc_wait_result(struct vmbus_softc *sc, struct vmbus_msghc *mh)
{
	struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;

	lwkt_gettoken(&mhc->mhc_active_token);

	KASSERT(mhc->mhc_active == mh, ("msghc mismatch"));
	while (mh->mh_resp == NULL)
		tsleep(&mhc->mhc_active, 0, "wmsghc", 0);
	mhc->mhc_active = NULL;

	lwkt_reltoken(&mhc->mhc_active_token);

	return mh->mh_resp;
}

void
vmbus_msghc_wakeup(struct vmbus_softc *sc, const struct vmbus_message *msg)
{
	struct vmbus_msghc_ctx *mhc = sc->vmbus_msg_hc;
	struct vmbus_msghc *mh;

	lwkt_gettoken(&mhc->mhc_active_token);

	mh = mhc->mhc_active;
	KASSERT(mh != NULL, ("no pending msg hypercall"));
	memcpy(&mh->mh_resp0, msg, sizeof(mh->mh_resp0));
	mh->mh_resp = &mh->mh_resp0;

	lwkt_reltoken(&mhc->mhc_active_token);
	wakeup(&mhc->mhc_active);
}

static int
vmbus_dma_alloc(struct vmbus_softc *sc)
{
	bus_dma_tag_t parent_dtag;
	uint8_t *evtflags;
	int cpu;

	parent_dtag = bus_get_dma_tag(sc->vmbus_dev);
	for (cpu = 0; cpu < ncpus; ++cpu) {
		struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);

		/*
		 * Per-cpu messages and event flags.
		 */
		psc->message = hyperv_dmamem_alloc(parent_dtag,
		    PAGE_SIZE, 0, PAGE_SIZE, &psc->message_dma,
		    BUS_DMA_WAITOK | BUS_DMA_ZERO);
		if (psc->message == NULL)
			return ENOMEM;

		psc->event_flags = hyperv_dmamem_alloc(parent_dtag,
		    PAGE_SIZE, 0, PAGE_SIZE, &psc->event_flags_dma,
		    BUS_DMA_WAITOK | BUS_DMA_ZERO);
		if (psc->event_flags == NULL)
			return ENOMEM;
	}

	evtflags = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	    PAGE_SIZE, &sc->vmbus_evtflags_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
	if (evtflags == NULL)
		return ENOMEM;
	sc->vmbus_rx_evtflags = (u_long *)evtflags;
	sc->vmbus_tx_evtflags = (u_long *)(evtflags + (PAGE_SIZE / 2));
	sc->vmbus_evtflags = evtflags;

	sc->vmbus_mnf1 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	    PAGE_SIZE, &sc->vmbus_mnf1_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
	if (sc->vmbus_mnf1 == NULL)
		return ENOMEM;

	sc->vmbus_mnf2 = hyperv_dmamem_alloc(parent_dtag, PAGE_SIZE, 0,
	    PAGE_SIZE, &sc->vmbus_mnf2_dma, BUS_DMA_WAITOK | BUS_DMA_ZERO);
	if (sc->vmbus_mnf2 == NULL)
		return ENOMEM;

	return 0;
}

static void
vmbus_dma_free(struct vmbus_softc *sc)
{
	int cpu;

	if (sc->vmbus_evtflags != NULL) {
		hyperv_dmamem_free(&sc->vmbus_evtflags_dma, sc->vmbus_evtflags);
		sc->vmbus_evtflags = NULL;
		sc->vmbus_rx_evtflags = NULL;
		sc->vmbus_tx_evtflags = NULL;
	}
	if (sc->vmbus_mnf1 != NULL) {
		hyperv_dmamem_free(&sc->vmbus_mnf1_dma, sc->vmbus_mnf1);
		sc->vmbus_mnf1 = NULL;
	}
	if (sc->vmbus_mnf2 != NULL) {
		hyperv_dmamem_free(&sc->vmbus_mnf2_dma, sc->vmbus_mnf2);
		sc->vmbus_mnf2 = NULL;
	}

	for (cpu = 0; cpu < ncpus; ++cpu) {
		struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);

		if (psc->message != NULL) {
			hyperv_dmamem_free(&psc->message_dma, psc->message);
			psc->message = NULL;
		}
		if (psc->event_flags != NULL) {
			hyperv_dmamem_free(&psc->event_flags_dma,
			    psc->event_flags);
			psc->event_flags = NULL;
		}
	}
}

static int
vmbus_intr_setup(struct vmbus_softc *sc)
{
	device_t dev = sc->vmbus_dev;
	device_t bus = device_get_parent(device_get_parent(dev));
	int rid, cpu;

	rid = 0;
	for (cpu = 0; cpu < ncpus; ++cpu) {
		struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);
		uint64_t msi_addr;
		uint32_t msi_data;
		int error;

		error = PCIB_ALLOC_MSIX(bus, dev, &psc->intr_irq, cpu);
		if (error) {
			device_printf(dev, "alloc vector on cpu%d failed: %d\n",
			    cpu, error);
			return ENXIO;
		}
		psc->intr_rid = ++rid;

		psc->intr_res = BUS_ALLOC_RESOURCE(bus, dev, SYS_RES_IRQ,
		    &psc->intr_rid, psc->intr_irq, psc->intr_irq, 1,
		    RF_ACTIVE, cpu);
		if (psc->intr_res == NULL) {
			device_printf(dev, "alloc irq on cpu%d failed: %d\n",
			    cpu, error);
			return ENXIO;
		}

		error = PCIB_MAP_MSI(bus, dev, rman_get_start(psc->intr_res),
		    &msi_addr, &msi_data, cpu);
		if (error) {
			device_printf(dev, "map irq on cpu%d failed: %d\n",
			    cpu, error);
			return ENXIO;
		}
		psc->intr_vec = hyperv_msi2vector(msi_addr, msi_data);

		if (bootverbose) {
			device_printf(dev, "vector %d irq %d on cpu%d\n",
			    psc->intr_vec, psc->intr_irq, cpu);
		}

		ksnprintf(psc->intr_desc, sizeof(psc->intr_desc), "%s cpu%d",
		    device_get_nameunit(dev), cpu);
		error = bus_setup_intr_descr(dev, psc->intr_res, INTR_MPSAFE,
		    vmbus_intr, psc, &psc->intr_hand, NULL, psc->intr_desc);
		if (error) {
			device_printf(dev, "setup intr on cpu%d failed: %d\n",
			    cpu, error);
			return ENXIO;
		}
	}
	return 0;
}

static void
vmbus_intr_teardown(struct vmbus_softc *sc)
{
	device_t dev = sc->vmbus_dev;
	device_t bus = device_get_parent(device_get_parent(dev));
	int cpu;

	for (cpu = 0; cpu < ncpus; ++cpu) {
		struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, cpu);

		if (psc->intr_hand != NULL) {
			bus_teardown_intr(dev, psc->intr_res, psc->intr_hand);
			psc->intr_hand = NULL;
		}

		if (psc->intr_res != NULL) {
			BUS_RELEASE_RESOURCE(bus, dev, SYS_RES_IRQ,
			    psc->intr_rid, psc->intr_res);
			psc->intr_res = NULL;
		}

		if (psc->intr_rid != 0) {
			PCIB_RELEASE_MSIX(bus, dev, psc->intr_irq, psc->cpuid);
			psc->intr_rid = 0;
		}
	}
}

static void
vmbus_synic_setup(void *xsc)
{
	struct vmbus_softc *sc = xsc;
	struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid);
	uint64_t val, orig;
	uint32_t sint;

	if (hyperv_features & CPUID_HV_MSR_VP_INDEX) {
		/*
		 * Save virtual processor id.
		 */
		psc->vcpuid = rdmsr(MSR_HV_VP_INDEX);
	} else {
		/*
		 * XXX
		 * Virtual processoor id is only used by a pretty broken
		 * channel selection code from storvsc.  It's nothing
		 * critical even if CPUID_HV_MSR_VP_INDEX is not set; keep
		 * moving on.
		 */
		psc->vcpuid = mycpuid;
	}

	/*
	 * Setup the SynIC message.
	 */
	orig = rdmsr(MSR_HV_SIMP);
	val = MSR_HV_SIMP_ENABLE | (orig & MSR_HV_SIMP_RSVD_MASK) |
	    ((psc->message_dma.hv_paddr >> PAGE_SHIFT) << MSR_HV_SIMP_PGSHIFT);
	wrmsr(MSR_HV_SIMP, val);

	/*
	 * Setup the SynIC event flags.
	 */
	orig = rdmsr(MSR_HV_SIEFP);
	val = MSR_HV_SIEFP_ENABLE | (orig & MSR_HV_SIEFP_RSVD_MASK) |
	    ((psc->event_flags_dma.hv_paddr >> PAGE_SHIFT) <<
	     MSR_HV_SIEFP_PGSHIFT);
	wrmsr(MSR_HV_SIEFP, val);


	/*
	 * Configure and unmask SINT for message and event flags.
	 */
	sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE;
	orig = rdmsr(sint);
	val = psc->intr_vec | /* MSR_HV_SINT_AUTOEOI | notyet */
	    (orig & MSR_HV_SINT_RSVD_MASK);
	wrmsr(sint, val);

	/*
	 * Configure and unmask SINT for timer.
	 */
	sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER;
	orig = rdmsr(sint);
	val = XTIMER_OFFSET | /* MSR_HV_SINT_AUTOEOI | notyet */
	    (orig & MSR_HV_SINT_RSVD_MASK);
	wrmsr(sint, val);

	/*
	 * All done; enable SynIC.
	 */
	orig = rdmsr(MSR_HV_SCONTROL);
	val = MSR_HV_SCTRL_ENABLE | (orig & MSR_HV_SCTRL_RSVD_MASK);
	wrmsr(MSR_HV_SCONTROL, val);
}

static void
vmbus_timer_stop(void *arg __unused)
{
	for (;;) {
		uint64_t val;

		/* Stop counting, and this also implies disabling STIMER0 */
		wrmsr(MSR_HV_STIMER0_COUNT, 0);

		val = rdmsr(MSR_HV_STIMER0_CONFIG);
		if ((val & MSR_HV_STIMER_CFG_ENABLE) == 0)
			break;
		cpu_pause();
	}
}

static void
vmbus_timer_config(void *arg __unused)
{
	/*
	 * Make sure that STIMER0 is really disabled before writing
	 * to STIMER0_CONFIG.
	 *
	 * "Writing to the configuration register of a timer that
	 *  is already enabled may result in undefined behaviour."
	 */
	vmbus_timer_stop(arg);
	wrmsr(MSR_HV_STIMER0_CONFIG,
	    MSR_HV_STIMER_CFG_AUTOEN | MSR_HV_STIMER0_CFG_SINT);
}

static void
vmbus_timer_msgintr(struct vmbus_pcpu_data *psc)
{
	volatile struct vmbus_message *msg;

	msg = psc->message + VMBUS_SINT_TIMER;
	if (msg->msg_type == HYPERV_MSGTYPE_TIMER_EXPIRED)
		vmbus_msg_reset(msg);
}

static void
vmbus_timer_restart(void *xsc)
{
	struct vmbus_softc *sc = xsc;
	struct vmbus_pcpu_data *psc = VMBUS_PCPU(sc, mycpuid);

	crit_enter();
	vmbus_timer_msgintr(psc);
	vmbus_timer_oneshot(psc, hyperv_tc64() + 1);
	crit_exit();
}

static void
vmbus_synic_teardown(void *arg __unused)
{
	uint64_t orig;
	uint32_t sint;

	/*
	 * Disable SynIC.
	 */
	orig = rdmsr(MSR_HV_SCONTROL);
	wrmsr(MSR_HV_SCONTROL, (orig & MSR_HV_SCTRL_RSVD_MASK));

	/*
	 * Mask message and event flags SINT.
	 */
	sint = MSR_HV_SINT0 + VMBUS_SINT_MESSAGE;
	orig = rdmsr(sint);
	wrmsr(sint, orig | MSR_HV_SINT_MASKED);

	/*
	 * Mask timer SINT.
	 */
	sint = MSR_HV_SINT0 + VMBUS_SINT_TIMER;
	orig = rdmsr(sint);
	wrmsr(sint, orig | MSR_HV_SINT_MASKED);

	/*
	 * Teardown SynIC message.
	 */
	orig = rdmsr(MSR_HV_SIMP);
	wrmsr(MSR_HV_SIMP, (orig & MSR_HV_SIMP_RSVD_MASK));

	/*
	 * Teardown SynIC event flags.
	 */
	orig = rdmsr(MSR_HV_SIEFP);
	wrmsr(MSR_HV_SIEFP, (orig & MSR_HV_SIEFP_RSVD_MASK));
}

static int
vmbus_init_contact(struct vmbus_softc *sc, uint32_t version)
{
	struct vmbus_chanmsg_init_contact *req;
	const struct vmbus_chanmsg_version_resp *resp;
	const struct vmbus_message *msg;
	struct vmbus_msghc *mh;
	int error, supp = 0;

	mh = vmbus_msghc_get(sc, sizeof(*req));
	if (mh == NULL)
		return ENXIO;

	req = vmbus_msghc_dataptr(mh);
	req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_INIT_CONTACT;
	req->chm_ver = version;
	req->chm_evtflags = sc->vmbus_evtflags_dma.hv_paddr;
	req->chm_mnf1 = sc->vmbus_mnf1_dma.hv_paddr;
	req->chm_mnf2 = sc->vmbus_mnf2_dma.hv_paddr;

	error = vmbus_msghc_exec(sc, mh);
	if (error) {
		vmbus_msghc_put(sc, mh);
		return error;
	}

	msg = vmbus_msghc_wait_result(sc, mh);
	resp = (const struct vmbus_chanmsg_version_resp *)msg->msg_data;
	supp = resp->chm_supp;

	vmbus_msghc_put(sc, mh);

	return (supp ? 0 : EOPNOTSUPP);
}

static int
vmbus_init(struct vmbus_softc *sc)
{
	int i;

	for (i = 0; i < nitems(vmbus_version); ++i) {
		int error;

		error = vmbus_init_contact(sc, vmbus_version[i]);
		if (!error) {
			sc->vmbus_version = vmbus_version[i];
			device_printf(sc->vmbus_dev, "version %u.%u\n",
			    (sc->vmbus_version >> 16),
			    (sc->vmbus_version & 0xffff));
			return 0;
		}
	}
	return ENXIO;
}

static void
vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg)
{
	const struct vmbus_chanmsg_hdr *hdr;

	hdr = (const struct vmbus_chanmsg_hdr *)msg->msg_data;

	/* TODO */
	if (hdr->chm_type == VMBUS_CHANMSG_TYPE_VERSION_RESP)
		vmbus_msghc_wakeup(sc, msg);
}