643 lines
18 KiB
C++
643 lines
18 KiB
C++
// vi:set ft=cpp: -*- Mode: C++ -*-
|
|
/* SPDX-License-Identifier: MIT */
|
|
/*
|
|
* (c) 2014 Alexander Warg <warg@os.inf.tu-dresden.de>
|
|
*/
|
|
|
|
#include <l4/re/util/debug>
|
|
#include <l4/sys/types.h>
|
|
#include <l4/sys/err.h>
|
|
#include <l4/cxx/bitfield>
|
|
#include <l4/cxx/exceptions>
|
|
#include <cstdint>
|
|
|
|
#pragma once
|
|
|
|
namespace L4virtio {
|
|
|
|
#if defined(__ARM_ARCH) && __ARM_ARCH == 7
|
|
static inline void wmb() { asm volatile ("dmb ishst" : : : "memory"); }
|
|
static inline void rmb() { asm volatile ("dmb ish" : : : "memory"); }
|
|
#elif defined(__ARM_ARCH) && __ARM_ARCH >= 8
|
|
static inline void wmb() { asm volatile ("dmb ishst" : : : "memory"); }
|
|
static inline void rmb() { asm volatile ("dmb ishld" : : : "memory"); }
|
|
#elif defined(__mips__)
|
|
static inline void wmb() { asm volatile ("sync" : : : "memory"); }
|
|
static inline void rmb() { asm volatile ("sync" : : : "memory"); }
|
|
#elif defined(__amd64__) || defined(__i386__) || defined(__i686__)
|
|
static inline void wmb() { asm volatile ("sfence" : : : "memory"); }
|
|
static inline void rmb() { asm volatile ("lfence" : : : "memory"); }
|
|
#elif defined(__riscv)
|
|
static inline void wmb() { asm volatile ("fence ow, ow" : : : "memory"); }
|
|
static inline void rmb() { asm volatile ("fence ir, ir" : : : "memory"); }
|
|
#else
|
|
#warning Missing proper memory write barrier
|
|
static inline void wmb() { asm volatile ("" : : : "memory"); }
|
|
static inline void rmb() { asm volatile ("" : : : "memory"); }
|
|
#endif
|
|
|
|
|
|
/**
|
|
* Pointer used in virtio descriptors.
|
|
*
|
|
* As the descriptor contain guest addresses these pointers cannot be
|
|
* dereferenced directly.
|
|
*/
|
|
template< typename T >
|
|
class Ptr
|
|
{
|
|
public:
|
|
/// Type for making an invalid (NULL) Ptr.
|
|
enum Invalid_type { Invalid /**< Use to set a Ptr to invalid (NULL) */ };
|
|
|
|
Ptr() = default;
|
|
|
|
/// Make and invalid Ptr
|
|
Ptr(Invalid_type) : _p(~0ULL) {}
|
|
|
|
/// Make a Ptr from a raw 64bit address
|
|
explicit Ptr(l4_uint64_t vm_addr) : _p(vm_addr) {}
|
|
|
|
/// \return The raw 64bit address of the stored pointer.
|
|
l4_uint64_t get() const { return _p; }
|
|
|
|
/// \return true if the stored pointer is valid (not NULL).
|
|
bool is_valid() const { return _p != ~0ULL; }
|
|
|
|
private:
|
|
l4_uint64_t _p;
|
|
};
|
|
|
|
|
|
/**
|
|
* Low-level Virtqueue.
|
|
*
|
|
* This class represents a single virtqueue, with a local running available
|
|
* index.
|
|
*
|
|
* \note The Virtqueue implementation is not thread-safe.
|
|
*/
|
|
class Virtqueue
|
|
{
|
|
public:
|
|
/**
|
|
* Descriptor in the descriptor table.
|
|
*/
|
|
class Desc
|
|
{
|
|
public:
|
|
/**
|
|
* Type for descriptor flags.
|
|
*/
|
|
struct Flags
|
|
{
|
|
l4_uint16_t raw; ///< raw flags value of a virtio descriptor.
|
|
Flags() = default;
|
|
|
|
/// Make Flags from raw 16bit value.
|
|
explicit Flags(l4_uint16_t v) : raw(v) {}
|
|
|
|
/// Part of a descriptor chain which is continued with the next field.
|
|
CXX_BITFIELD_MEMBER( 0, 0, next, raw);
|
|
/// Block described by this descriptor is writeable.
|
|
CXX_BITFIELD_MEMBER( 1, 1, write, raw);
|
|
/// Indirect descriptor, block contains a list of descriptors.
|
|
CXX_BITFIELD_MEMBER( 2, 2, indirect, raw);
|
|
};
|
|
|
|
Ptr<void> addr; ///< Address stored in descriptor.
|
|
l4_uint32_t len; ///< Length of described buffer.
|
|
Flags flags; ///< Descriptor flags.
|
|
l4_uint16_t next; ///< Index of the next chained descriptor.
|
|
|
|
/**
|
|
* Dump a single descriptor.
|
|
*/
|
|
void dump(unsigned idx) const
|
|
{
|
|
L4Re::Util::Dbg().printf("D[%04x]: %08llx (%x) f=%04x n=%04x\n",
|
|
idx, addr.get(),
|
|
len, static_cast<unsigned>(flags.raw),
|
|
static_cast<unsigned>(next));
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Type of available ring, this is read-only for the host.
|
|
*/
|
|
class Avail
|
|
{
|
|
public:
|
|
/**
|
|
* Flags of the available ring.
|
|
*/
|
|
struct Flags
|
|
{
|
|
l4_uint16_t raw; ///< raw 16bit flags value of the available ring.
|
|
Flags() = default;
|
|
|
|
/// Make Flags from the raw value.
|
|
explicit Flags(l4_uint16_t v) : raw(v) {}
|
|
|
|
/// Guest does not want to receive interrupts when requests are finished.
|
|
CXX_BITFIELD_MEMBER( 0, 0, no_irq, raw);
|
|
};
|
|
|
|
Flags flags; ///< flags of available ring
|
|
l4_uint16_t idx; ///< available index written by guest
|
|
l4_uint16_t ring[]; ///< array of available descriptor indexes.
|
|
};
|
|
|
|
/**
|
|
* Type of an element of the used ring.
|
|
*/
|
|
struct Used_elem
|
|
{
|
|
Used_elem() = default;
|
|
|
|
/**
|
|
* Initialize a used ring element.
|
|
*
|
|
* \param id The index of the descriptor to be marked as used.
|
|
* \param len The total bytes written into the buffer of the
|
|
* descriptor chain.
|
|
*/
|
|
Used_elem(l4_uint16_t id, l4_uint32_t len) : id(id), len(len) {}
|
|
l4_uint32_t id; ///< descriptor index
|
|
l4_uint32_t len; ///< length field
|
|
};
|
|
|
|
/**
|
|
* Used ring.
|
|
*/
|
|
class Used
|
|
{
|
|
public:
|
|
/**
|
|
* flags for the used ring.
|
|
*/
|
|
struct Flags
|
|
{
|
|
l4_uint16_t raw; ///< raw flags value as specified by virtio.
|
|
Flags() = default;
|
|
|
|
/// make Flags from raw value
|
|
explicit Flags(l4_uint16_t v) : raw(v) {}
|
|
|
|
/// host does not want to be notified when new requests have been queued.
|
|
CXX_BITFIELD_MEMBER( 0, 0, no_notify, raw);
|
|
};
|
|
|
|
Flags flags; ///< flags of the used ring.
|
|
l4_uint16_t idx; ///< index of the last entry in the ring.
|
|
Used_elem ring[]; ///< array of used descriptors.
|
|
};
|
|
|
|
protected:
|
|
Desc *_desc = nullptr; ///< pointer to descriptor table, NULL if queue is off.
|
|
Avail *_avail = nullptr; ///< pointer to available ring.
|
|
Used *_used = nullptr; ///< pointer to used ring.
|
|
|
|
/** The life counter for the queue */
|
|
l4_uint16_t _current_avail = 0;
|
|
|
|
/**
|
|
* mask used for indexing into the descriptor table
|
|
* and the rings.
|
|
*/
|
|
l4_uint16_t _idx_mask = 0;
|
|
|
|
/**
|
|
* Create a disabled virtqueue.
|
|
*/
|
|
Virtqueue() = default;
|
|
|
|
Virtqueue(Virtqueue const &) = delete;
|
|
~Virtqueue() = default;
|
|
|
|
public:
|
|
/**
|
|
* Completely disable the queue.
|
|
*
|
|
* setup() must be used to enable the queue again.
|
|
*/
|
|
void disable()
|
|
{ _desc = 0; }
|
|
|
|
/**
|
|
* Fixed alignment values for different parts of a virtqueue.
|
|
*/
|
|
enum
|
|
{
|
|
Desc_align = 4, //< Alignment of the descriptor table.
|
|
Avail_align = 1, //< Alignment of the available ring.
|
|
Used_align = 2, //< Alignment of the used ring.
|
|
};
|
|
|
|
/**
|
|
* Calculate the total size for a virtqueue of the given dimensions.
|
|
*
|
|
* \param num The number of entries in the descriptor table, the
|
|
* available ring, and the used ring (must be a power of 2).
|
|
*
|
|
* \return The total size in bytes of the queue data structures.
|
|
*/
|
|
static unsigned long total_size(unsigned num)
|
|
{
|
|
static_assert(Desc_align >= Avail_align,
|
|
"virtqueue alignment assumptions broken");
|
|
return l4_round_size(desc_size(num) + avail_size(num), Used_align)
|
|
+ used_size(num);
|
|
}
|
|
|
|
/**
|
|
* Calculate the size of the descriptor table for `num` entries.
|
|
*
|
|
* \param num The number of entries in the descriptor table.
|
|
*
|
|
* \returns The size in bytes needed for a descriptor table with
|
|
* `num` entries.
|
|
*/
|
|
static unsigned long desc_size(unsigned num)
|
|
{ return num * 16; }
|
|
|
|
/**
|
|
* Get the alignment in zero LSBs needed for the descriptor table.
|
|
*
|
|
* \returns The alignment in zero LSBs needed for a descriptor table.
|
|
*/
|
|
static unsigned long desc_align()
|
|
{ return Desc_align; }
|
|
|
|
/**
|
|
* Calculate the size of the available ring for `num` entries.
|
|
*
|
|
* \param num The number of entries in the available ring.
|
|
* \returns The size in bytes needed for an available ring with
|
|
* `num` entries.
|
|
*/
|
|
static unsigned long avail_size(unsigned num)
|
|
{ return 2 * num + 6; }
|
|
|
|
/**
|
|
* Get the alignment in zero LSBs needed for the available ring.
|
|
*
|
|
* \returns The alignment in zero LSBs needed for an available ring.
|
|
*/
|
|
static unsigned long avail_align()
|
|
{ return Avail_align; }
|
|
|
|
/**
|
|
* Calculate the size of the used ring for `num` entries.
|
|
*
|
|
* \param num The number of entries in the used ring.
|
|
*
|
|
* \returns The size in bytes needed for an used ring with
|
|
* `num` entries.
|
|
*/
|
|
static unsigned long used_size(unsigned num)
|
|
{ return 8 * num + 6; }
|
|
|
|
/**
|
|
* Get the alignment in zero LSBs needed for the used ring.
|
|
*
|
|
* \returns The alignment in zero LSBs needed for an used ring.
|
|
*/
|
|
static unsigned long used_align()
|
|
{ return Used_align; }
|
|
|
|
/**
|
|
* Calculate the total size of this virtqueue.
|
|
*
|
|
* \pre The queue has been set up.
|
|
*/
|
|
unsigned long total_size() const
|
|
{
|
|
return (reinterpret_cast<char *>(_used) - reinterpret_cast<char *>(_desc))
|
|
+ used_size(num());
|
|
}
|
|
|
|
/**
|
|
* Get the offset of the available ring from the descriptor table.
|
|
*/
|
|
unsigned long avail_offset() const
|
|
{ return reinterpret_cast<char *>(_avail) - reinterpret_cast<char *>(_desc); }
|
|
|
|
/**
|
|
* Get the offset of the used ring from the descriptor table.
|
|
*/
|
|
unsigned long used_offset() const
|
|
{ return reinterpret_cast<char *>(_used) - reinterpret_cast<char *>(_desc); }
|
|
|
|
/**
|
|
* Enable this queue.
|
|
*
|
|
* \param num The number of entries in the descriptor table, the
|
|
* available ring, and the used ring (must be a power of 2).
|
|
* \param desc The address of the descriptor table. (Must be
|
|
* Desc_align aligned and at least `desc_size(num)` bytes
|
|
* in size.)
|
|
* \param avail The address of the available ring. (Must be
|
|
* Avail_align aligned and at least `avail_size(num)` bytes
|
|
* in size.)
|
|
* \param used The address of the used ring. (Must be Used_align aligned
|
|
* and at least `used_size(num)` bytes in size.)
|
|
*
|
|
* Due to the data type of the descriptors, the queue can have a
|
|
* maximum size of 2^16.
|
|
*/
|
|
void setup(unsigned num, void *desc, void *avail, void *used)
|
|
{
|
|
if (num > 0x10000)
|
|
throw L4::Runtime_error(-L4_EINVAL, "Queue too large.");
|
|
|
|
_idx_mask = num - 1;
|
|
_desc = static_cast<Desc*>(desc);
|
|
_avail = static_cast<Avail*>(avail);
|
|
_used = static_cast<Used*>(used);
|
|
|
|
_current_avail = 0;
|
|
|
|
L4Re::Util::Dbg().printf("VQ[%p]: num=%d d:%p a:%p u:%p\n",
|
|
this, num, _desc, _avail, _used);
|
|
}
|
|
|
|
/**
|
|
* Enable this queue.
|
|
*
|
|
* \param num The number of entries in the descriptor table, the
|
|
* available ring, and the used ring (must be a power of 2).
|
|
* \param ring The base address for the queue data structure. The memory
|
|
* block at `ring` must be at least `total_size(num)` bytes
|
|
* in size and have an alignment of Desc_align
|
|
* (desc_align()) bits.
|
|
*
|
|
* Due to the data type of the descriptors, the queue can have a
|
|
* maximum size of 2^16.
|
|
*/
|
|
void setup_simple(unsigned num, void *ring)
|
|
{
|
|
l4_addr_t desc = reinterpret_cast<l4_addr_t>(ring);
|
|
l4_addr_t avail = l4_round_size(desc + desc_size(num), Avail_align);
|
|
void *used = reinterpret_cast<void *>(
|
|
l4_round_size(avail + avail_size(num), Used_align));
|
|
setup(num, ring, reinterpret_cast<void *>(avail), used);
|
|
}
|
|
|
|
/**
|
|
* Dump descriptors for this queue.
|
|
*
|
|
* \pre the queue must be in working state.
|
|
*/
|
|
void dump(Desc const *d) const
|
|
{ d->dump(d - _desc); }
|
|
|
|
/**
|
|
* Test if this queue is in working state.
|
|
*
|
|
* \return true when the queue is in working state, false else.
|
|
*/
|
|
bool ready() const
|
|
{ return L4_LIKELY(_desc != 0); }
|
|
|
|
/// \return The number of entries in the ring.
|
|
unsigned num() const
|
|
{ return _idx_mask + 1; }
|
|
|
|
/**
|
|
* Get the no IRQ flag of this queue.
|
|
*
|
|
* \pre queue must be in working state.
|
|
*
|
|
* \return true if the guest does not want to get IRQs (currently).
|
|
*/
|
|
bool no_notify_guest() const
|
|
{
|
|
return _avail->flags.no_irq();
|
|
}
|
|
|
|
/**
|
|
* Get the no notify flag of this queue.
|
|
*
|
|
* \pre queue must be in working state.
|
|
*
|
|
* \return true if the host does not want to get IRQs (currently).
|
|
*/
|
|
bool no_notify_host() const
|
|
{
|
|
return _used->flags.no_notify();
|
|
}
|
|
|
|
/**
|
|
* Set the no-notify flag for this queue
|
|
*
|
|
* \pre Queue must be in a working state.
|
|
*/
|
|
void no_notify_host(bool value)
|
|
{
|
|
_used->flags.no_notify() = value;
|
|
}
|
|
|
|
/**
|
|
* Get available index from available ring (for debugging).
|
|
*
|
|
* \pre Queue must be in a working state.
|
|
*
|
|
* \return current index in the available ring (shared
|
|
* between device model and device driver).
|
|
*/
|
|
l4_uint16_t get_avail_idx() const { return _avail->idx; }
|
|
|
|
/**
|
|
* Get tail-available index stored in local state (for debugging).
|
|
*
|
|
* \return current tail index for the the available ring.
|
|
*/
|
|
l4_uint16_t get_tail_avail_idx() const { return _current_avail; }
|
|
|
|
};
|
|
|
|
namespace Driver {
|
|
|
|
/**
|
|
* Driver-side implementation of a Virtqueue.
|
|
*
|
|
* Adds function for managing the descriptor list, enqueueing new
|
|
* and dequeueing finished requests.
|
|
*
|
|
* \note The Virtqueue implementation is not thread-safe.
|
|
*/
|
|
class Virtqueue : public L4virtio::Virtqueue
|
|
{
|
|
private:
|
|
/// Index of next free entry in the descriptor table.
|
|
l4_uint16_t _next_free;
|
|
|
|
public:
|
|
enum End_of_queue
|
|
{
|
|
// Indicates the end of the queue.
|
|
Eoq = 0xFFFF
|
|
};
|
|
|
|
Virtqueue() : _next_free(Eoq) {}
|
|
|
|
/**
|
|
* Initialize the descriptor table and the index structures
|
|
* of this queue.
|
|
*
|
|
* \param num The number of entries in the descriptor table, the
|
|
* available ring, and the used ring (must be a power of 2).
|
|
*
|
|
* \pre The queue must be set up correctly with setup() or setup_simple().
|
|
*/
|
|
void initialize_rings(unsigned num)
|
|
{
|
|
_used->idx = 0;
|
|
_avail->idx = 0;
|
|
|
|
// setup the freelist
|
|
for (l4_uint16_t d = 0; d < num - 1; ++d)
|
|
_desc[d].next = d + 1;
|
|
_desc[num - 1].next = Eoq;
|
|
_next_free = 0;
|
|
}
|
|
|
|
/**
|
|
* Initialize this virtqueue.
|
|
*
|
|
* \param num The number of entries in the descriptor table, the
|
|
* available ring, and the used ring (must be a power of 2).
|
|
* \param desc The address of the descriptor table. (Must be
|
|
* Desc_align aligned and at least `desc_size(num)` bytes
|
|
* in size.)
|
|
* \param avail The address of the available ring. (Must be
|
|
* Avail_align aligned and at least `avail_size(num)` bytes
|
|
* in size.)
|
|
* \param used The address of the used ring. (Must be Used_align aligned
|
|
* and at least `used_size(num)` bytes in size.)
|
|
*
|
|
* This function sets up the memory and initializes the freelist.
|
|
*/
|
|
void init_queue(unsigned num, void *desc, void *avail, void *used)
|
|
{
|
|
setup(num, desc, avail, used);
|
|
initialize_rings(num);
|
|
}
|
|
|
|
/**
|
|
* Initialize this virtqueue.
|
|
*
|
|
* \param num The number of entries in the descriptor table, the
|
|
* available ring, and the used ring (must be a power of 2).
|
|
* \param base The base address for the queue data structure.
|
|
*
|
|
* This function sets up the memory and initializes the freelist.
|
|
*/
|
|
void init_queue(unsigned num, void *base)
|
|
{
|
|
setup_simple(num, base);
|
|
initialize_rings(num);
|
|
}
|
|
|
|
|
|
/**
|
|
* Allocate and return an unused descriptor from the descriptor table.
|
|
*
|
|
* The descriptor will be removed from the free list, the content
|
|
* should be considered undefined. After use, it needs to be freed
|
|
* using free_descriptor().
|
|
*
|
|
* \return The index of the reserved descriptor or Virtqueue::Eoq if
|
|
* no free descriptor is available.
|
|
*
|
|
* Note: the implementation uses (2^16 - 1) as the end of queue marker.
|
|
* That means that the final entry in the queue can not be allocated
|
|
* iff the queue size is 2^16.
|
|
*/
|
|
l4_uint16_t alloc_descriptor()
|
|
{
|
|
l4_uint16_t idx = _next_free;
|
|
if (idx == Eoq)
|
|
return Eoq;
|
|
|
|
_next_free = _desc[idx].next;
|
|
|
|
return idx;
|
|
}
|
|
|
|
/**
|
|
* Enqueue a descriptor in the available ring.
|
|
*
|
|
* \param descno Index of the head descriptor to enqueue.
|
|
*/
|
|
void enqueue_descriptor(l4_uint16_t descno)
|
|
{
|
|
if (descno > _idx_mask)
|
|
throw L4::Bounds_error();
|
|
|
|
_avail->ring[_avail->idx & _idx_mask] = descno; // _avail->idx expected to wrap
|
|
wmb();
|
|
++_avail->idx;
|
|
}
|
|
|
|
/**
|
|
* Return a reference to a descriptor in the descriptor table.
|
|
*
|
|
* \param descno Index of the descriptor,
|
|
* expected to be in correct range.
|
|
*/
|
|
Desc &desc(l4_uint16_t descno)
|
|
{
|
|
if (descno > _idx_mask)
|
|
throw L4::Bounds_error();
|
|
|
|
return _desc[descno];
|
|
}
|
|
|
|
/**
|
|
* Return the next finished block.
|
|
*
|
|
* \param[out] len (optional) Size of valid data in finished block.
|
|
* Note that this is the value reported by the device,
|
|
* which may set it to a value that is larger than the
|
|
* original buffer size.
|
|
*
|
|
* \return Index of the head or Virtqueue::Eoq
|
|
* if no used element is currently available.
|
|
*/
|
|
l4_uint16_t find_next_used(l4_uint32_t *len = nullptr)
|
|
{
|
|
if (_current_avail == _used->idx)
|
|
return Eoq;
|
|
|
|
auto elem = _used->ring[_current_avail++ & _idx_mask];
|
|
|
|
if (len)
|
|
*len = elem.len;
|
|
|
|
return elem.id;
|
|
}
|
|
|
|
/**
|
|
* Free a chained list of descriptors in the descriptor queue.
|
|
*
|
|
* \param head Index of the first element in the descriptor chain.
|
|
* \param tail Index of the last element in the descriptor chain.
|
|
*
|
|
* Simply takes the descriptor chain and prepends it to the beginning
|
|
* of the free list. Assumes that the list has been correctly chained.
|
|
*/
|
|
void free_descriptor(l4_uint16_t head, l4_uint16_t tail)
|
|
{
|
|
if (head > _idx_mask || tail > _idx_mask)
|
|
throw L4::Bounds_error();
|
|
|
|
_desc[tail].next = _next_free;
|
|
_next_free = head;
|
|
}
|
|
};
|
|
|
|
}
|
|
} // namespace L4virtio
|