// SPDX-License-Identifier: GPL-2.0-or-later AND BSD-3-Clause
/*
 * virtio API, vring and virtqueue functions definition
 *
 * Copyright Red Hat
 * Author: Laurent Vivier <lvivier@redhat.com>
 */

/* Some parts copied from QEMU subprojects/libvhost-user/libvhost-user.c
 * originally licensed under the following terms:
 *
 * --
 *
 * Copyright IBM, Corp. 2007
 * Copyright (c) 2016 Red Hat, Inc.
 *
 * Authors:
 *  Anthony Liguori <aliguori@us.ibm.com>
 *  Marc-André Lureau <mlureau@redhat.com>
 *  Victor Kaplansky <victork@redhat.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or
 * later.  See the COPYING file in the top-level directory.
 *
 * Some parts copied from QEMU hw/virtio/virtio.c
 * licensed under the following terms:
 *
 * Copyright IBM, Corp. 2007
 *
 * Authors:
 *  Anthony Liguori   <aliguori@us.ibm.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 * --
 *
 * virtq_used_event() and virtq_avail_event() from
 * https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-712000A
 * licensed under the following terms:
 *
 * --
 *
 * This header is BSD licensed so anyone can use the definitions
 * to implement compatible drivers/servers.
 *
 * Copyright 2007, 2009, IBM Corporation
 * Copyright 2011, Red Hat, Inc
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of IBM nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ‘‘AS IS’’ AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "passt.h"
#include <assert.h>
#include <limits.h>
#include <stddef.h>
#include <endian.h>
#include <string.h>
#include <errno.h>
#include <sys/eventfd.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <limits.h>
#include <stdint.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdlib.h>
#include <sys/eventfd.h>
#include <netinet/in.h>

#include "util.h"
#include "virtio.h"
#include "vhost_user.h"
#include "tcp_buf.h"
#include "epoll_ctl.h"
#include "udp.h"


struct vq_state vqs[2];

struct vring_desc vring_desc[2][VHOST_NDESCS] __attribute__((aligned(PAGE_SIZE)));

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
union vring_avail_u vring_avail_all[2] __attribute__((aligned(PAGE_SIZE)));
union vring_used_u vring_used_all[2] __attribute__((aligned(PAGE_SIZE)));
#pragma GCC diagnostic pop

union vhost_memory_u vhost_memory = {
	.mem = {
		.nregions = N_VHOST_REGIONS,
	},
};

/**
 * vu_gpa_to_va() - Translate guest physical address to our virtual address.
 * @dev:	Vhost-user device
 * @plen:	Physical length to map (input), capped to region (output)
 * @guest_addr:	Guest physical address
 *
 * Return: virtual address in our address space of the guest physical address
 */
static void *vu_gpa_to_va(const struct vu_dev *dev, uint64_t *plen,
			  uint64_t guest_addr)
{
	unsigned int i;

	if (*plen == 0)
		return NULL;

	/* Find matching memory region. */
	for (i = 0; i < dev->memory.nregions; i++) {
		const struct vu_dev_region *r = &dev->memory.regions[i];

		if ((guest_addr >= r->gpa) &&
		    (guest_addr < (r->gpa + r->size))) {
			if ((guest_addr + *plen) > (r->gpa + r->size))
				*plen = r->gpa + r->size - guest_addr;
			/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
			return (void *)(uintptr_t)(guest_addr - r->gpa +
						   r->mmap_addr +
						   r->mmap_offset);
		}
	}

	return NULL;
}

/**
 * vring_avail_flags() - Read the available ring flags
 * @vq:		Virtqueue
 *
 * Return: the available ring descriptor flags of the given virtqueue
 */
static inline uint16_t vring_avail_flags(const struct vu_virtq *vq)
{
	return le16toh(vq->vring.avail->flags);
}

/**
 * vring_avail_idx() - Read the available ring index
 * @vq:		Virtqueue
 *
 * Return: the available ring index of the given virtqueue
 */
static inline uint16_t vring_avail_idx(struct vu_virtq *vq)
{
	vq->shadow_avail_idx = le16toh(vq->vring.avail->idx);

	return vq->shadow_avail_idx;
}

/**
 * vring_avail_ring() - Read an available ring entry
 * @vq:		Virtqueue
 * @i:		Index of the entry to read
 *
 * Return: the ring entry content (head of the descriptor chain)
 */
static inline uint16_t vring_avail_ring(const struct vu_virtq *vq, int i)
{
	return le16toh(vq->vring.avail->ring[i]);
}

/**
 * virtq_used_event() - Get location of used event indices
 *		      (only with VIRTIO_F_EVENT_IDX)
 * @vq:		Virtqueue
 *
 * Return: return the location of the used event index
 */
static inline uint16_t *virtq_used_event(const struct vu_virtq *vq)
{
	/* For backwards compat, used event index is at *end* of avail ring. */
	return &vq->vring.avail->ring[vq->vring.num];
}

/**
 * vring_get_used_event() - Get the used event from the available ring
 * @vq:		Virtqueue
 *
 * Return: the used event (available only if VIRTIO_RING_F_EVENT_IDX is set)
 *         used_event is a performant alternative where the driver
 *         specifies how far the device can progress before a notification
 *         is required.
 */
static inline uint16_t vring_get_used_event(const struct vu_virtq *vq)
{
	return le16toh(*virtq_used_event(vq));
}

/**
 * virtqueue_get_head() - Get the head of the descriptor chain for a given
 *                        index
 * @vq:		Virtqueue
 * @idx:	Available ring entry index
 * @head:	Head of the descriptor chain
 */
static void virtqueue_get_head(const struct vu_virtq *vq,
			       unsigned int idx, unsigned int *head)
{
	/* Grab the next descriptor number they're advertising, and increment
	 * the index we've seen.
	 */
	*head = vring_avail_ring(vq, idx % vq->vring.num);

	/* If their number is silly, that's a fatal mistake. */
	if (*head >= vq->vring.num)
		die("vhost-user: Guest says index %u is available", *head);
}

/**
 * virtqueue_read_indirect_desc() - Copy virtio ring descriptors from guest
 *                                  memory
 * @dev:	Vhost-user device
 * @desc:	Destination address to copy the descriptors to
 * @addr:	Guest memory address to copy from
 * @len:	Length of memory to copy
 *
 * Return: -1 if there is an error, 0 otherwise
 */
static int virtqueue_read_indirect_desc(const struct vu_dev *dev,
					struct vring_desc *desc,
					uint64_t addr, size_t len)
{
	uint64_t read_len;

	if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc)))
		return -1;

	if (len == 0)
		return -1;

	while (len) {
		const struct vring_desc *orig_desc;

		read_len = len;
		orig_desc = vu_gpa_to_va(dev, &read_len, addr);
		if (!orig_desc)
			return -1;

		memcpy(desc, orig_desc, read_len);
		len -= read_len;
		addr += read_len;
		/* NOLINTNEXTLINE(bugprone-sizeof-expression,cert-arr39-c) */
		desc += read_len / sizeof(struct vring_desc);
	}

	return 0;
}

/**
 * enum virtqueue_read_desc_state - State in the descriptor chain
 * @VIRTQUEUE_READ_DESC_ERROR:	Found an invalid descriptor
 * @VIRTQUEUE_READ_DESC_DONE:	No more descriptors in the chain
 * @VIRTQUEUE_READ_DESC_MORE:	there are more descriptors in the chain
 */
enum virtqueue_read_desc_state {
	VIRTQUEUE_READ_DESC_ERROR = -1,
	VIRTQUEUE_READ_DESC_DONE = 0,   /* end of chain */
	VIRTQUEUE_READ_DESC_MORE = 1,   /* more buffers in chain */
};

/**
 * virtqueue_read_next_desc() - Read the the next descriptor in the chain
 * @desc:	Virtio ring descriptors
 * @i:		Index of the current descriptor
 * @max:	Maximum value of the descriptor index
 * @next:	Index of the next descriptor in the chain (output value)
 *
 * Return: current chain descriptor state (error, next, done)
 */
static int virtqueue_read_next_desc(const struct vring_desc *desc,
				    int i, unsigned int max, unsigned int *next)
{
	/* If this descriptor says it doesn't chain, we're done. */
	if (!(le16toh(desc[i].flags) & VRING_DESC_F_NEXT))
		return VIRTQUEUE_READ_DESC_DONE;

	/* Check they're not leading us off end of descriptors. */
	*next = le16toh(desc[i].next);
	/* Make sure compiler knows to grab that: we don't want it changing! */
	smp_wmb();

	if (*next >= max)
		return VIRTQUEUE_READ_DESC_ERROR;

	return VIRTQUEUE_READ_DESC_MORE;
}

/**
 * vu_queue_empty() - Check if virtqueue is empty
 * @vq:		Virtqueue
 *
 * Return: true if the virtqueue is empty, false otherwise
 */
static bool vu_queue_empty(struct vu_virtq *vq)
{
	if (!vq->vring.avail)
		return true;

	if (vq->shadow_avail_idx != vq->last_avail_idx)
		return false;

	return vring_avail_idx(vq) == vq->last_avail_idx;
}

/**
 * vring_can_notify() - Check if a notification can be sent
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 *
 * Return: true if notification can be sent
 */
static bool vring_can_notify(const struct vu_dev *dev, struct vu_virtq *vq)
{
	uint16_t old, new;
	bool v;

	/* We need to expose used array entries before checking used event. */
	smp_mb();

	/* Always notify when queue is empty (when feature acknowledge) */
	if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
	    !vq->inuse && vu_queue_empty(vq))
		return true;

	if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX))
		return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);

	v = vq->signalled_used_valid;
	vq->signalled_used_valid = true;
	old = vq->signalled_used;
	new = vq->signalled_used = vq->used_idx;
	return !v || vring_need_event(vring_get_used_event(vq), new, old);
}

/**
 * vu_queue_notify() - Send a notification to the given virtqueue
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 */
void vu_queue_notify(const struct vu_dev *dev, struct vu_virtq *vq)
{
	if (!vq->vring.avail)
		return;

	if (!vring_can_notify(dev, vq)) {
		debug("vhost-user: virtqueue can skip notify...");
		return;
	}

	if (eventfd_write(vq->call_fd, 1) < 0)
		die_perror("Error writing vhost-user queue eventfd");
}

/**
 * virtq_avail_event() -  Get location of available event indices
 *			  (only with VIRTIO_F_EVENT_IDX)
 * @vq:		Virtqueue
 *
 * Return: return the location of the available event index
 */
static inline uint16_t *virtq_avail_event(const struct vu_virtq *vq)
{
	/* For backwards compat, avail event index is at *end* of used ring. */
	return (uint16_t *)&vq->vring.used->ring[vq->vring.num];
}

/**
 * vring_set_avail_event() - Set avail_event
 * @vq:		Virtqueue
 * @val:	Value to set to avail_event
 *		avail_event is used in the same way the used_event is in the
 *		avail_ring.
 *		avail_event is used to advise the driver that notifications
 *		are unnecessary until the driver writes entry with an index
 *		specified by avail_event into the available ring.
 */
static inline void vring_set_avail_event(const struct vu_virtq *vq,
					 uint16_t val)
{
	uint16_t val_le = htole16(val);

	if (!vq->notification)
		return;

	memcpy(virtq_avail_event(vq), &val_le, sizeof(val_le));
}

/**
 * virtqueue_map_desc() - Translate descriptor ring physical address into our
 * 			  virtual address space
 * @dev:	Vhost-user device
 * @p_num_sg:	First iov entry to use (input),
 *		first iov entry not used (output)
 * @iov:	Iov array to use to store buffer virtual addresses
 * @max_num_sg:	Maximum number of iov entries
 * @pa:		Guest physical address of the buffer to map into our virtual
 * 		address
 * @sz:		Size of the buffer
 *
 * Return: false on error, true otherwise
 */
static bool virtqueue_map_desc(const struct vu_dev *dev,
			       unsigned int *p_num_sg, struct iovec *iov,
			       unsigned int max_num_sg,
			       uint64_t pa, size_t sz)
{
	unsigned int num_sg = *p_num_sg;

	assert(num_sg < max_num_sg);
	assert(sz);

	while (sz) {
		uint64_t len = sz;

		iov[num_sg].iov_base = vu_gpa_to_va(dev, &len, pa);
		if (iov[num_sg].iov_base == NULL)
			die("vhost-user: invalid address for buffers");
		iov[num_sg].iov_len = len;
		num_sg++;
		sz -= len;
		pa += len;
	}

	*p_num_sg = num_sg;
	return true;
}

/**
 * vu_queue_map_desc() - Map the virtqueue descriptor ring into our virtual
 * 			 address space
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @idx:	First descriptor ring entry to map
 * @elem:	Virtqueue element to store descriptor ring iov
 * @in_sg:	Incoming iovec array for device-writable descriptors
 * @max_in_sg:	Maximum number of entries in @in_sg
 * @out_sg:	Outgoing iovec array for device-readable descriptors
 * @max_out_sg:	Maximum number of entries in @out_sg
 *
 * Return: -1 if there is an error, 0 otherwise
 */
static int vu_queue_map_desc(const struct vu_dev *dev,
			     struct vu_virtq *vq, unsigned int idx,
			     struct vu_virtq_element *elem,
			     struct iovec *in_sg, size_t max_in_sg,
			     struct iovec *out_sg, size_t max_out_sg)
{
	const struct vring_desc *desc = vq->vring.desc;
	struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
	unsigned int out_num = 0, in_num = 0;
	unsigned int max = vq->vring.num;
	unsigned int i = idx;
	uint64_t read_len;
	int rc;

	if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
		unsigned int desc_len;
		uint64_t desc_addr;

		if (le32toh(desc[i].len) % sizeof(struct vring_desc))
			die("vhost-user: Invalid size for indirect buffer table");

		/* loop over the indirect descriptor table */
		desc_addr = le64toh(desc[i].addr);
		desc_len = le32toh(desc[i].len);
		max = desc_len / sizeof(struct vring_desc);
		read_len = desc_len;
		desc = vu_gpa_to_va(dev, &read_len, desc_addr);
		if (desc && read_len != desc_len) {
			/* Failed to use zero copy */
			desc = NULL;
			if (!virtqueue_read_indirect_desc(dev, desc_buf, desc_addr, desc_len))
				desc = desc_buf;
		}
		if (!desc)
			die("vhost-user: Invalid indirect buffer table");
		i = 0;
	}

	/* Collect all the descriptors */
	do {
		if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
			if (!virtqueue_map_desc(dev, &in_num, in_sg,
						max_in_sg,
						le64toh(desc[i].addr),
						le32toh(desc[i].len)))
				return -1;
		} else {
			if (in_num)
				die("Incorrect order for descriptors");
			if (!virtqueue_map_desc(dev, &out_num, out_sg,
						max_out_sg,
						le64toh(desc[i].addr),
						le32toh(desc[i].len))) {
				return -1;
			}
		}

		/* If we've got too many, that implies a descriptor loop. */
		if ((in_num + out_num) > max)
			die("vhost-user: Loop in queue descriptor list");
		rc = virtqueue_read_next_desc(desc, i, max, &i);
	} while (rc == VIRTQUEUE_READ_DESC_MORE);

	if (rc == VIRTQUEUE_READ_DESC_ERROR)
		die("vhost-user: Failed to read descriptor list");

	elem->index = idx;
	elem->in_sg = in_sg;
	elem->in_num = in_num;
	elem->out_sg = out_sg;
	elem->out_num = out_num;

	return 0;
}

/**
 * vu_queue_pop() - Pop an entry from the virtqueue
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @elem:	Virtqueue element to fill with the entry information
 * @in_sg:	Incoming iovec array for device-writable descriptors
 * @max_in_sg:	Maximum number of entries in @in_sg
 * @out_sg:	Outgoing iovec array for device-readable descriptors
 * @max_out_sg:	Maximum number of entries in @out_sg
 *
 * Return: -1 if there is an error, 0 otherwise
 */
int vu_queue_pop(const struct vu_dev *dev, struct vu_virtq *vq,
		 struct vu_virtq_element *elem,
		 struct iovec *in_sg, size_t max_in_sg,
		 struct iovec *out_sg, size_t max_out_sg)
{
	unsigned int head;
	int ret;

	if (!vq->vring.avail)
		return -1;

	if (vu_queue_empty(vq))
		return -1;

	/* Needed after vu_queue_empty(), see comment in
	 * virtqueue_num_heads().
	 */
	smp_rmb();

	if (vq->inuse >= vq->vring.num)
		die("vhost-user queue size exceeded");

	virtqueue_get_head(vq, vq->last_avail_idx++, &head);

	if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX))
		vring_set_avail_event(vq, vq->last_avail_idx);

	ret = vu_queue_map_desc(dev, vq, head, elem, in_sg, max_in_sg,
				out_sg, max_out_sg);

	if (ret < 0)
		return ret;

	vq->inuse++;

	return 0;
}

/**
 * vu_queue_detach_element() - Detach an element from the virtqueue
 * @vq:		Virtqueue
 */
void vu_queue_detach_element(struct vu_virtq *vq)
{
	vq->inuse--;
	/* unmap, when DMA support is added */
}

/**
 * vu_queue_unpop() - Push back the previously popped element from the virtqueue
 * @vq:		Virtqueue
 */
/* cppcheck-suppress unusedFunction */
void vu_queue_unpop(struct vu_virtq *vq)
{
	vq->last_avail_idx--;
	vu_queue_detach_element(vq);
}

/**
 * vu_queue_rewind() - Push back a given number of popped elements
 * @vq:		Virtqueue
 * @num:	Number of element to unpop
 *
 * Return: true on success, false if not
 */
bool vu_queue_rewind(struct vu_virtq *vq, unsigned int num)
{
	if (num > vq->inuse)
		return false;

	vq->last_avail_idx -= num;
	vq->inuse -= num;
	return true;
}

/**
 * vring_used_write() - Write an entry in the used ring
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @uelem:	Entry to write
 * @i:		Index of the entry in the used ring
 */
static inline void vring_used_write(const struct vu_dev *vdev,
				    struct vu_virtq *vq,
				    const struct vring_used_elem *uelem, int i)
{
	struct vring_used *used = vq->vring.used;

	used->ring[i] = *uelem;
	vu_log_write(vdev, vq->vring.log_guest_addr +
		     offsetof(struct vring_used, ring[i]),
		     sizeof(used->ring[i]));
}

/**
 * vu_log_queue_fill() - Log virtqueue memory update
 * @dev:	vhost-user device
 * @vq:		Virtqueue
 * @index:	Descriptor ring index
 * @len:	Size of the element
 */
static void vu_log_queue_fill(const struct vu_dev *vdev, struct vu_virtq *vq,
			      unsigned int index, unsigned int len)
{
	struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
	struct vring_desc *desc = vq->vring.desc;
	unsigned num_bufs = 0;
	uint64_t read_len;
	unsigned int max;

	if (!vdev->log_table || !len || !vu_has_feature(vdev, VHOST_F_LOG_ALL))
		return;

	max = vq->vring.num;

	if (le16toh(desc[index].flags) & VRING_DESC_F_INDIRECT) {
		unsigned int desc_len;
		uint64_t desc_addr;

		if (le32toh(desc[index].len) % sizeof(struct vring_desc))
			die("Invalid size for indirect buffer table");

		/* loop over the indirect descriptor table */
		desc_addr = le64toh(desc[index].addr);
		desc_len = le32toh(desc[index].len);
		max = desc_len / sizeof(struct vring_desc);
		read_len = desc_len;
		desc = vu_gpa_to_va(vdev, &read_len, desc_addr);
		if (desc && read_len != desc_len) {
			/* Failed to use zero copy */
			desc = NULL;
			if (!virtqueue_read_indirect_desc(vdev, desc_buf,
							  desc_addr,
							  desc_len))
				desc = desc_buf;
		}

		if (!desc)
			die("Invalid indirect buffer table");

		index = 0;
	}

	do {
		if (++num_bufs > max)
			die("Looped descriptor");

		if (le16toh(desc[index].flags) & VRING_DESC_F_WRITE) {
			unsigned min = MIN(le32toh(desc[index].len), len);
			vu_log_write(vdev, le64toh(desc[index].addr), min);
			len -= min;
		}
	} while (len > 0 &&
		 (virtqueue_read_next_desc(desc, index, max, &index) ==
		  VIRTQUEUE_READ_DESC_MORE));
}


/**
 * vu_queue_fill_by_index() - Update information of a descriptor ring entry
 *			      in the used ring
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @index:	Descriptor ring index
 * @len:	Size of the element
 * @idx:	Used ring entry index
 */
static void vu_queue_fill_by_index(const struct vu_dev *vdev,
				   struct vu_virtq *vq,
				   unsigned int index, unsigned int len,
				   unsigned int idx)
{
	struct vring_used_elem uelem;

	if (!vq->vring.avail)
		return;

	vu_log_queue_fill(vdev, vq, index, len);

	idx = (idx + vq->used_idx) % vq->vring.num;

	uelem.id = htole32(index);
	uelem.len = htole32(len);
	vring_used_write(vdev, vq, &uelem, idx);
}

/**
 * vu_queue_fill() - Update information of a given element in the used ring
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @elem:	Element information to fill
 * @len:	Size of the element
 * @idx:	Used ring entry index
 */
void vu_queue_fill(const struct vu_dev *vdev, struct vu_virtq *vq,
		   const struct vu_virtq_element *elem, unsigned int len,
		   unsigned int idx)
{
	vu_queue_fill_by_index(vdev, vq, elem->index, len, idx);
}

/**
 * vring_used_idx_set() - Set the descriptor ring current index
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @val:	Value to set in the index
 */
static inline void vring_used_idx_set(const struct vu_dev *vdev,
				      struct vu_virtq *vq, uint16_t val)
{
	vq->vring.used->idx = htole16(val);
	vu_log_write(vdev, vq->vring.log_guest_addr +
		     offsetof(struct vring_used, idx),
		     sizeof(vq->vring.used->idx));

	vq->used_idx = val;
}

/**
 * vu_queue_flush() - Flush the virtqueue
 * @dev:	Vhost-user device
 * @vq:		Virtqueue
 * @count:	Number of entry to flush
 */
void vu_queue_flush(const struct vu_dev *vdev, struct vu_virtq *vq,
		    unsigned int count)
{
	uint16_t old, new;

	if (!vq->vring.avail)
		return;

	/* Make sure buffer is written before we update index. */
	smp_wmb();

	old = vq->used_idx;
	new = old + count;
	vring_used_idx_set(vdev, vq, new);
	vq->inuse -= count;
	if ((uint16_t)(new - vq->signalled_used) < (uint16_t)(new - old))
		vq->signalled_used_valid = false;
}

/**
 * setup_vhost_net() - Open and negotiate features on /dev/vhost-net
 * @c:			Execution context; c->fd_vhost is set on success
 *
 */
void setup_vhost_net(struct ctx *c)
{
	static const uint64_t req_features =
		(1ULL << VIRTIO_F_VERSION_1) | (1ULL << VHOST_NET_F_VIRTIO_NET_HDR);
	int vhost_fd, rc;

	vhost_fd = open("/dev/vhost-net", O_RDWR | O_NONBLOCK | O_CLOEXEC);
	if (vhost_fd < 0)
		die_perror("Failed to open /dev/vhost-net");

	rc = ioctl(vhost_fd, VHOST_SET_OWNER, NULL);
	if (rc < 0)
		die_perror("VHOST_SET_OWNER ioctl on /dev/vhost-net failed");

	rc = ioctl(vhost_fd, VHOST_GET_FEATURES, &c->virtio_features);
	if (rc < 0)
		die_perror("VHOST_GET_FEATURES ioctl on /dev/vhost-net failed");

	debug("vhost features: %lx", c->virtio_features);
	debug("req features: %lx", req_features);

	c->virtio_features &= req_features;
	if (c->virtio_features != req_features)
		die("vhost does not support required features");

	rc = ioctl(vhost_fd, VHOST_SET_FEATURES, &c->virtio_features);
	if (rc < 0)
		die_perror("VHOST_SET_FEATURES ioctl on /dev/vhost-net failed");

	c->fd_vhost = vhost_fd;
}

/**
 * setup_eventfds() - Set up call/kick eventfds and vring size for one queue
 * @c:		Execution context; c->fd_vhost must already be set
 * @queue_idx:	Index of the queue (vring) to configure
 *
 */
void setup_eventfds(struct ctx *c, int queue_idx)
{
	int vhost_fd = c->fd_vhost;
	struct vhost_vring_file call_file = { .index = queue_idx };
	struct vhost_vring_file kick_file = { .index = queue_idx };
	struct vhost_vring_file err_file = { .index = queue_idx };

	struct vhost_vring_state state = {
		.index = queue_idx,
		.num = VHOST_NDESCS,
	};
	union epoll_ref ref = {
		.type = EPOLL_TYPE_VHOST_CALL,
		.queue = queue_idx,
	};
	struct epoll_event ev;
	int rc;

	call_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
	if (call_file.fd < 0)
		die_perror("Failed to create call eventfd");
	ref.fd = call_file.fd;

	rc = ioctl(vhost_fd, VHOST_SET_VRING_CALL, &call_file);
	if (rc < 0)
		die_perror("VHOST_SET_VRING_CALL ioctl on /dev/vhost-net failed");

	ev = (struct epoll_event){ .data.u64 = ref.u64, .events = EPOLLIN };
	rc = epoll_ctl(c->epollfd, EPOLL_CTL_ADD, ref.fd, &ev);
	if (rc < 0)
		die_perror("Failed to add call eventfd to epoll");
	c->vq[queue_idx].call_fd = call_file.fd;

	err_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
	if (err_file.fd < 0)
		die_perror("Failed to create error eventfd");

	rc = ioctl(vhost_fd, VHOST_SET_VRING_ERR, &err_file);
	if (rc < 0)
		die_perror("VHOST_SET_VRING_ERR ioctl on /dev/vhost-net failed");

	ref.type = EPOLL_TYPE_VHOST_ERROR;
	ref.fd = err_file.fd;
	ev.data.u64 = ref.u64;
	rc = epoll_ctl(c->epollfd, EPOLL_CTL_ADD, ref.fd, &ev);
	if (rc < 0)
		die_perror("Failed to add error eventfd to epoll");
	c->vq[queue_idx].err_fd = err_file.fd;

	rc = ioctl(vhost_fd, VHOST_SET_VRING_NUM, &state);
	if (rc < 0) {
		die_perror("VHOST_SET_VRING_NUM ioctl on /dev/vhost-net failed (queue %d)",
			    queue_idx);
	}

	kick_file.fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
	if (kick_file.fd < 0)
		die_perror("Failed to create kick eventfd");

	rc = ioctl(vhost_fd, VHOST_SET_VRING_KICK, &kick_file);
	if (rc < 0) {
		die_perror("VHOST_SET_VRING_KICK ioctl on /dev/vhost-net failed (queue %d)",
			    queue_idx);
	}

	c->vq[queue_idx].kick_fd = kick_file.fd;

	vqs[queue_idx].num_free = VHOST_NDESCS;
}

/**
 * setup_memory_table() - Register the GPA/HVA translation table
 * @c:		Execution context; c->fd_vhost must already be set
 *
 * pasta has no real guest, so container->host addresses are 1:1 and can
 * be interpreted directly rather than translated.
 */
int setup_memory_table(struct ctx *c) {
#define VHOST_MEMORY_REGION_PTR(addr, size) \
    (struct vhost_memory_region) { \
        .guest_phys_addr = (uintptr_t)addr, \
        .memory_size = size, \
        .userspace_addr  = (uintptr_t)addr, \
    }
#define VHOST_MEMORY_REGION(elem) VHOST_MEMORY_REGION_PTR(&elem, sizeof(elem))

	/* general purpose buffers */
    vhost_memory.mem.regions[0] = VHOST_MEMORY_REGION(pkt_buf);
	vhost_memory.mem.regions[1] = VHOST_MEMORY_REGION(eth_pad);

	/* tcp specific buffers */
   	vhost_memory.mem.regions[2] = VHOST_MEMORY_REGION(tcp_payload_tap_hdr);
	vhost_memory.mem.regions[3] = VHOST_MEMORY_REGION(tcp4_payload_ip);
	vhost_memory.mem.regions[4] = VHOST_MEMORY_REGION(tcp6_payload_ip);
	vhost_memory.mem.regions[5] = VHOST_MEMORY_REGION(tcp_payload);
	vhost_memory.mem.regions[6] = VHOST_MEMORY_REGION(tcp_eth_hdr);

	/* udp specific buffers */
	vhost_memory.mem.regions[7] = VHOST_MEMORY_REGION(udp_payload);
	vhost_memory.mem.regions[8] = VHOST_MEMORY_REGION(udp_eth_hdr);
	vhost_memory.mem.regions[9] = VHOST_MEMORY_REGION(udp_iov_recv);
	vhost_memory.mem.regions[10] = VHOST_MEMORY_REGION(udp_mh_recv);
	vhost_memory.mem.regions[11] = VHOST_MEMORY_REGION(udp_meta);
	vhost_memory.mem.nregions = 12;

	return ioctl(c->fd_vhost, VHOST_SET_MEM_TABLE, &vhost_memory.mem);
}



/**
 * set_vring_for_queue() - Register a vring's addresses and bind its backend
 * @c:		Execution context; c->fd_vhost must already be set
 * @queue_idx:	Index of the queue (vring) to configure
 * @tap_fd:	Tap fd to bind as this queue's backend
 *
 */
void set_vring_for_queue(struct ctx *c, int queue_idx, int tap_fd)
{
	int vhost_fd = c->fd_vhost;
	struct vhost_vring_addr addr = {
		.index = queue_idx,
		.desc_user_addr = (unsigned long)vring_desc[queue_idx],
		.avail_user_addr = (unsigned long)&vring_avail_all[queue_idx],
		.used_user_addr = (unsigned long)&vring_used_all[queue_idx],
		.log_guest_addr = (unsigned long)&vring_used_all[queue_idx],
	};
	struct vhost_vring_file file = {
		.index = queue_idx,
		.fd = tap_fd,
	};
	unsigned int i;
	int rc;

	debug("qid: %d", queue_idx);
	debug("vhost desc addr: 0x%llx", addr.desc_user_addr);
	debug("vhost avail addr: 0x%llx", addr.avail_user_addr);
	debug("vhost used addr: 0x%llx", addr.used_user_addr);

	rc = ioctl(vhost_fd, VHOST_SET_VRING_ADDR, &addr);
	if (rc < 0)
		die_perror("VHOST_SET_VRING_ADDR ioctl on /dev/vhost-net failed");

	if (queue_idx == 0) {
		for (i = 0; i < VHOST_NDESCS; ++i) {
			vring_desc[0][i].addr = (uintptr_t)pkt_buf +
				i * (PKT_BUF_BYTES / VHOST_NDESCS);
			vring_desc[0][i].len = PKT_BUF_BYTES / VHOST_NDESCS;
			vring_desc[0][i].flags = VRING_DESC_F_WRITE;
		}

		for (i = 0; i < VHOST_NDESCS; ++i)
			vring_avail_all[0].avail.ring[i] = htole16(i);

		rx_descriptor_handoff(c);
	}

	if (queue_idx == 1) {
		for (i = 0; i < (VHOST_NDESCS - 1); ++i) {
			vring_desc[1][i].next = i+1;
		}
	}

	debug("qid: %d", file.index);
	debug("tap fd: %d", file.fd);
	rc = ioctl(vhost_fd, VHOST_NET_SET_BACKEND, &file);
	if (rc < 0)
		die_perror("VHOST_NET_SET_BACKEND ioctl on /dev/vhost-net failed");
}

/**
 * rx_descriptor_handoff() - Batch-announce freed RX descriptors to the kernel
 * @c:		Execution context
 *
 * Bumps avail.idx by the number of descriptors accumulated in
 * vqs[0].num_free (from prior consume_one_rx_descriptor() calls),
 * then resets the counter to zero. The kernel will see the new
 * avail.idx and consume the freshly-available descriptors.
 *
 */
void rx_descriptor_handoff(struct ctx *c)
{
	smp_wmb();

	if (!vqs[0].num_free)
		return;

	vring_avail_all[0].avail.idx += vqs[0].num_free;
	vqs[0].num_free = 0;
	vhost_kick(&vring_used_all[0].used, c->vq[0].kick_fd);
}


/**
 * vhost_kick() - Notify the kernel that new descriptors are available
 * @used:		The vring_used queue. Taken as a parameter to check if the kernel
 * 				virtio thread is actively reading descriptors or no
 * @kick_fd:	Which fd to notify about (will differ by which queue we are about
 * 				to announce availability in) 
 */
void vhost_kick(struct vring_used *used, int kick_fd) {
	/* Ensure that the read to used->flags doesn't get reordered to be
	* above the avail.idx update 
	*/
	smp_mb();

	if (!(used->flags & VRING_USED_F_NO_NOTIFY))
		eventfd_write(kick_fd, 1);
}