Loading...
Searching...
No Matches
virtqueue
// vi:set ft=cpp: -*- Mode: C++ -*-
/* SPDX-License-Identifier: MIT */
/*
* (c) 2014 Alexander Warg <warg@os.inf.tu-dresden.de>
*/
#include <l4/re/util/debug>
#include <l4/sys/types.h>
#include <l4/sys/err.h>
#include <l4/cxx/bitfield>
#include <l4/cxx/exceptions>
#include <cstdint>
#pragma once
namespace L4virtio {
#if defined(__ARM_ARCH) && __ARM_ARCH == 7
static inline void wmb() { asm volatile ("dmb ishst" : : : "memory"); }
static inline void rmb() { asm volatile ("dmb ish" : : : "memory"); }
#elif defined(__ARM_ARCH) && __ARM_ARCH >= 8
static inline void wmb() { asm volatile ("dmb ishst" : : : "memory"); }
static inline void rmb() { asm volatile ("dmb ishld" : : : "memory"); }
#elif defined(__mips__)
static inline void wmb() { asm volatile ("sync" : : : "memory"); }
static inline void rmb() { asm volatile ("sync" : : : "memory"); }
#elif defined(__amd64__) || defined(__i386__) || defined(__i686__)
static inline void wmb() { asm volatile ("sfence" : : : "memory"); }
static inline void rmb() { asm volatile ("lfence" : : : "memory"); }
#elif defined(__riscv)
static inline void wmb() { asm volatile ("fence ow, ow" : : : "memory"); }
static inline void rmb() { asm volatile ("fence ir, ir" : : : "memory"); }
#else
#warning Missing proper memory write barrier
static inline void wmb() { asm volatile ("" : : : "memory"); }
static inline void rmb() { asm volatile ("" : : : "memory"); }
#endif
template< typename T >
class Ptr
{
public:
enum Invalid_type { Invalid };
Ptr() = default;
Ptr(Invalid_type) : _p(~0ULL) {}
explicit Ptr(l4_uint64_t vm_addr) : _p(vm_addr) {}
l4_uint64_t get() const { return _p; }
bool is_valid() const { return _p != ~0ULL; }
private:
l4_uint64_t _p;
};
class Virtqueue
{
public:
class Desc
{
public:
struct Flags
{
l4_uint16_t raw;
Flags() = default;
explicit Flags(l4_uint16_t v) : raw(v) {}
CXX_BITFIELD_MEMBER( 0, 0, next, raw);
CXX_BITFIELD_MEMBER( 1, 1, write, raw);
CXX_BITFIELD_MEMBER( 2, 2, indirect, raw);
};
Ptr<void> addr;
l4_uint32_t len;
Flags flags;
l4_uint16_t next;
void dump(unsigned idx) const
{
L4Re::Util::Dbg().printf("D[%04x]: %08llx (%x) f=%04x n=%04x\n",
idx, addr.get(),
len, static_cast<unsigned>(flags.raw),
static_cast<unsigned>(next));
}
};
class Avail
{
public:
struct Flags
{
l4_uint16_t raw;
Flags() = default;
explicit Flags(l4_uint16_t v) : raw(v) {}
CXX_BITFIELD_MEMBER( 0, 0, no_irq, raw);
};
Flags flags;
l4_uint16_t idx;
l4_uint16_t ring[];
};
struct Used_elem
{
Used_elem() = default;
Used_elem(l4_uint16_t id, l4_uint32_t len) : id(id), len(len) {}
l4_uint32_t id;
l4_uint32_t len;
};
class Used
{
public:
struct Flags
{
l4_uint16_t raw;
Flags() = default;
explicit Flags(l4_uint16_t v) : raw(v) {}
CXX_BITFIELD_MEMBER( 0, 0, no_notify, raw);
};
Flags flags;
l4_uint16_t idx;
Used_elem ring[];
};
protected:
Desc *_desc = nullptr;
Avail *_avail = nullptr;
Used *_used = nullptr;
l4_uint16_t _current_avail = 0;
l4_uint16_t _idx_mask = 0;
Virtqueue() = default;
Virtqueue(Virtqueue const &) = delete;
~Virtqueue() = default;
public:
void disable()
{ _desc = 0; }
enum
{
Desc_align = 4, //< Alignment of the descriptor table.
Avail_align = 1, //< Alignment of the available ring.
Used_align = 2, //< Alignment of the used ring.
};
static unsigned long total_size(unsigned num)
{
static_assert(Desc_align >= Avail_align,
"virtqueue alignment assumptions broken");
return l4_round_size(desc_size(num) + avail_size(num), Used_align)
+ used_size(num);
}
static unsigned long desc_size(unsigned num)
{ return num * 16; }
static unsigned long desc_align()
{ return Desc_align; }
static unsigned long avail_size(unsigned num)
{ return 2 * num + 6; }
static unsigned long avail_align()
{ return Avail_align; }
static unsigned long used_size(unsigned num)
{ return 8 * num + 6; }
static unsigned long used_align()
{ return Used_align; }
unsigned long total_size() const
{
return (reinterpret_cast<char *>(_used) - reinterpret_cast<char *>(_desc))
+ used_size(num());
}
unsigned long avail_offset() const
{ return reinterpret_cast<char *>(_avail) - reinterpret_cast<char *>(_desc); }
unsigned long used_offset() const
{ return reinterpret_cast<char *>(_used) - reinterpret_cast<char *>(_desc); }
void setup(unsigned num, void *desc, void *avail, void *used)
{
if (num > 0x10000)
throw L4::Runtime_error(-L4_EINVAL, "Queue too large.");
_idx_mask = num - 1;
_desc = static_cast< Desc*>(desc);
_avail = static_cast< Avail*>(avail);
_used = static_cast< Used*>(used);
_current_avail = 0;
L4Re::Util::Dbg().printf("VQ[%p]: num=%d d:%p a:%p u:%p\n",
this, num, _desc, _avail, _used);
}
void setup_simple(unsigned num, void *ring)
{
l4_addr_t desc = reinterpret_cast< l4_addr_t >(ring);
l4_addr_t avail = l4_round_size(desc + desc_size(num), Avail_align);
void *used = reinterpret_cast<void *>(
l4_round_size(avail + avail_size(num), Used_align));
setup(num, ring, reinterpret_cast<void *>(avail), used);
}
void dump(Desc const *d) const
{ d->dump(d - _desc); }
bool ready() const
{ return L4_LIKELY(_desc != 0); }
unsigned num() const
{ return _idx_mask + 1; }
bool no_notify_guest() const
{
return _avail->flags.no_irq();
}
bool no_notify_host() const
{
return _used->flags.no_notify();
}
void no_notify_host(bool value)
{
_used->flags.no_notify() = value;
}
l4_uint16_t get_avail_idx() const { return _avail->idx; }
l4_uint16_t get_tail_avail_idx() const { return _current_avail; }
};
namespace Driver {
class Virtqueue : public L4virtio::Virtqueue
{
private:
l4_uint16_t _next_free;
public:
enum End_of_queue
{
// Indicates the end of the queue.
Eoq = 0xFFFF
};
Virtqueue() : _next_free(Eoq) {}
void initialize_rings(unsigned num)
{
_used->idx = 0;
_avail->idx = 0;
// setup the freelist
for (l4_uint16_t d = 0; d < num - 1; ++d)
_desc[d].next = d + 1;
_desc[num - 1].next = Eoq;
_next_free = 0;
}
void init_queue(unsigned num, void *desc, void *avail, void *used)
{
setup(num, desc, avail, used);
initialize_rings(num);
}
void init_queue(unsigned num, void *base)
{
setup_simple(num, base);
initialize_rings(num);
}
l4_uint16_t alloc_descriptor()
{
l4_uint16_t idx = _next_free;
if (idx == Eoq)
return Eoq;
_next_free = _desc[idx].next;
return idx;
}
void enqueue_descriptor(l4_uint16_t descno)
{
if (descno > _idx_mask)
throw L4::Bounds_error();
_avail->ring[_avail->idx & _idx_mask] = descno; // _avail->idx expected to wrap
wmb();
++_avail->idx;
}
Desc &desc(l4_uint16_t descno)
{
if (descno > _idx_mask)
throw L4::Bounds_error();
return _desc[descno];
}
l4_uint16_t find_next_used(l4_uint32_t *len = nullptr)
{
if (_current_avail == _used->idx)
return Eoq;
auto elem = _used->ring[_current_avail++ & _idx_mask];
if (len)
*len = elem.len;
return elem.id;
}
void free_descriptor(l4_uint16_t head, l4_uint16_t tail)
{
if (head > _idx_mask || tail > _idx_mask)
throw L4::Bounds_error();
_desc[tail].next = _next_free;
_next_free = head;
}
};
}
} // namespace L4virtio
L4::Bounds_error
Access out of bounds.
Definition exceptions:279
L4::Runtime_error
Exception for an abstract runtime error.
Definition exceptions:129
L4virtio::Driver::Virtqueue::free_descriptor
void free_descriptor(l4_uint16_t head, l4_uint16_t tail)
Free a chained list of descriptors in the descriptor queue.
Definition virtqueue:631
L4virtio::Driver::Virtqueue::enqueue_descriptor
void enqueue_descriptor(l4_uint16_t descno)
Enqueue a descriptor in the available ring.
Definition virtqueue:574
L4virtio::Driver::Virtqueue::init_queue
void init_queue(unsigned num, void *base)
Initialize this virtqueue.
Definition virtqueue:537
L4virtio::Driver::Virtqueue::alloc_descriptor
l4_uint16_t alloc_descriptor()
Allocate and return an unused descriptor from the descriptor table.
Definition virtqueue:558
L4virtio::Driver::Virtqueue::find_next_used
l4_uint16_t find_next_used(l4_uint32_t *len=nullptr)
Return the next finished block.
Definition virtqueue:609
L4virtio::Driver::Virtqueue::init_queue
void init_queue(unsigned num, void *desc, void *avail, void *used)
Initialize this virtqueue.
Definition virtqueue:522
L4virtio::Driver::Virtqueue::desc
Desc & desc(l4_uint16_t descno)
Return a reference to a descriptor in the descriptor table.
Definition virtqueue:590
L4virtio::Driver::Virtqueue::initialize_rings
void initialize_rings(unsigned num)
Initialize the descriptor table and the index structures of this queue.
Definition virtqueue:494
L4virtio::Ptr
Pointer used in virtio descriptors.
Definition virtqueue:48
L4virtio::Ptr::Ptr
Ptr(l4_uint64_t vm_addr)
Make a Ptr from a raw 64bit address.
Definition virtqueue:59
L4virtio::Ptr::get
l4_uint64_t get() const
Definition virtqueue:62
L4virtio::Ptr::Invalid_type
Invalid_type
Type for making an invalid (NULL) Ptr.
Definition virtqueue:51
L4virtio::Ptr::Invalid
@ Invalid
Use to set a Ptr to invalid (NULL).
Definition virtqueue:51
L4virtio::Ptr::is_valid
bool is_valid() const
Definition virtqueue:65
L4virtio::Ptr::Ptr
Ptr(Invalid_type)
Make and invalid Ptr.
Definition virtqueue:56
L4virtio::Virtqueue::Avail
Type of available ring, this is read-only for the host.
Definition virtqueue:129
L4virtio::Virtqueue::Avail::ring
l4_uint16_t ring[]
array of available descriptor indexes.
Definition virtqueue:148
L4virtio::Virtqueue::Avail::flags
Flags flags
flags of available ring
Definition virtqueue:146
L4virtio::Virtqueue::Avail::idx
l4_uint16_t idx
available index written by guest
Definition virtqueue:147
L4virtio::Virtqueue::Desc
Descriptor in the descriptor table.
Definition virtqueue:87
L4virtio::Virtqueue::Desc::next
l4_uint16_t next
Index of the next chained descriptor.
Definition virtqueue:111
L4virtio::Virtqueue::Desc::len
l4_uint32_t len
Length of described buffer.
Definition virtqueue:109
L4virtio::Virtqueue::Desc::flags
Flags flags
Descriptor flags.
Definition virtqueue:110
L4virtio::Virtqueue::Desc::dump
void dump(unsigned idx) const
Dump a single descriptor.
Definition virtqueue:116
L4virtio::Virtqueue::Desc::addr
Ptr< void > addr
Address stored in descriptor.
Definition virtqueue:108
L4virtio::Virtqueue::Used
Used ring.
Definition virtqueue:174
L4virtio::Virtqueue::Used::ring
Used_elem ring[]
array of used descriptors.
Definition virtqueue:193
L4virtio::Virtqueue::Used::idx
l4_uint16_t idx
index of the last entry in the ring.
Definition virtqueue:192
L4virtio::Virtqueue::Used::flags
Flags flags
flags of the used ring.
Definition virtqueue:191
L4virtio::Virtqueue
Low-level Virtqueue.
Definition virtqueue:81
L4virtio::Virtqueue::no_notify_host
void no_notify_host(bool value)
Set the no-notify flag for this queue.
Definition virtqueue:436
L4virtio::Virtqueue::disable
void disable()
Completely disable the queue.
Definition virtqueue:224
L4virtio::Virtqueue::setup
void setup(unsigned num, void *desc, void *avail, void *used)
Enable this queue.
Definition virtqueue:349
L4virtio::Virtqueue::desc_size
static unsigned long desc_size(unsigned num)
Calculate the size of the descriptor table for num entries.
Definition virtqueue:261
L4virtio::Virtqueue::get_tail_avail_idx
l4_uint16_t get_tail_avail_idx() const
Get tail-available index stored in local state (for debugging).
Definition virtqueue:456
L4virtio::Virtqueue::_used
Used * _used
pointer to used ring.
Definition virtqueue:199
L4virtio::Virtqueue::no_notify_guest
bool no_notify_guest() const
Get the no IRQ flag of this queue.
Definition virtqueue:414
L4virtio::Virtqueue::avail_align
static unsigned long avail_align()
Get the alignment in zero LSBs needed for the available ring.
Definition virtqueue:287
L4virtio::Virtqueue::dump
void dump(Desc const *d) const
Dump descriptors for this queue.
Definition virtqueue:392
L4virtio::Virtqueue::setup_simple
void setup_simple(unsigned num, void *ring)
Enable this queue.
Definition virtqueue:378
L4virtio::Virtqueue::avail_offset
unsigned long avail_offset() const
Get the offset of the available ring from the descriptor table.
Definition virtqueue:323
L4virtio::Virtqueue::used_align
static unsigned long used_align()
Get the alignment in zero LSBs needed for the used ring.
Definition virtqueue:306
L4virtio::Virtqueue::total_size
static unsigned long total_size(unsigned num)
Calculate the total size for a virtqueue of the given dimensions.
Definition virtqueue:245
L4virtio::Virtqueue::desc_align
static unsigned long desc_align()
Get the alignment in zero LSBs needed for the descriptor table.
Definition virtqueue:269
L4virtio::Virtqueue::used_size
static unsigned long used_size(unsigned num)
Calculate the size of the used ring for num entries.
Definition virtqueue:298
L4virtio::Virtqueue::Virtqueue
Virtqueue()=default
Create a disabled virtqueue.
L4virtio::Virtqueue::used_offset
unsigned long used_offset() const
Get the offset of the used ring from the descriptor table.
Definition virtqueue:329
L4virtio::Virtqueue::avail_size
static unsigned long avail_size(unsigned num)
Calculate the size of the available ring for num entries.
Definition virtqueue:279
L4virtio::Virtqueue::total_size
unsigned long total_size() const
Calculate the total size of this virtqueue.
Definition virtqueue:314
L4virtio::Virtqueue::ready
bool ready() const
Test if this queue is in working state.
Definition virtqueue:400
L4virtio::Virtqueue::_idx_mask
l4_uint16_t _idx_mask
mask used for indexing into the descriptor table and the rings.
Definition virtqueue:208
L4virtio::Virtqueue::_desc
Desc * _desc
pointer to descriptor table, NULL if queue is off.
Definition virtqueue:197
L4virtio::Virtqueue::get_avail_idx
l4_uint16_t get_avail_idx() const
Get available index from available ring (for debugging).
Definition virtqueue:449
L4virtio::Virtqueue::no_notify_host
bool no_notify_host() const
Get the no notify flag of this queue.
Definition virtqueue:426
L4virtio::Virtqueue::_avail
Avail * _avail
pointer to available ring.
Definition virtqueue:198
L4virtio::Virtqueue::_current_avail
l4_uint16_t _current_avail
The life counter for the queue.
Definition virtqueue:202
L4virtio::Virtqueue::num
unsigned num() const
Definition virtqueue:404
err.h
Error codes.
exceptions
Base exceptions.
l4_addr_t
unsigned long l4_addr_t
Address type.
Definition l4int.h:34
l4_uint32_t
unsigned int l4_uint32_t
Unsigned 32bit value.
Definition l4int.h:29
l4_uint16_t
unsigned short int l4_uint16_t
Unsigned 16bit value.
Definition l4int.h:27
l4_uint64_t
unsigned long long l4_uint64_t
Unsigned 64bit value.
Definition l4int.h:31
L4_EINVAL
@ L4_EINVAL
Invalid argument.
Definition err.h:46
l4_round_size
l4_addr_t l4_round_size(l4_addr_t value, unsigned char bits) L4_NOTHROW
Round value up to the next alignment with bits size.
Definition consts.h:484
L4_LIKELY
#define L4_LIKELY(x)
Expression is likely to execute.
Definition compiler.h:283
types.h
Common L4 ABI Data Types.
L4virtio
L4-VIRTIO Transport C++ API.
Definition l4virtio:26
L4virtio::Virtqueue::Avail::Flags
Flags of the available ring.
Definition virtqueue:135
L4virtio::Virtqueue::Avail::Flags::no_irq
constexpr no_irq_bfm_t::Val no_irq() const
Get the no_irq bits (0 to 0) of raw.
Definition virtqueue:143
L4virtio::Virtqueue::Avail::Flags::Flags
Flags(l4_uint16_t v)
Make Flags from the raw value.
Definition virtqueue:140
L4virtio::Virtqueue::Avail::Flags::raw
l4_uint16_t raw
raw 16bit flags value of the available ring.
Definition virtqueue:136
L4virtio::Virtqueue::Desc::Flags
Type for descriptor flags.
Definition virtqueue:93
L4virtio::Virtqueue::Desc::Flags::next
constexpr next_bfm_t::Val next() const
Get the next bits (0 to 0) of raw.
Definition virtqueue:101
L4virtio::Virtqueue::Desc::Flags::write
constexpr write_bfm_t::Val write() const
Get the write bits (1 to 1) of raw.
Definition virtqueue:103
L4virtio::Virtqueue::Desc::Flags::Flags
Flags(l4_uint16_t v)
Make Flags from raw 16bit value.
Definition virtqueue:98
L4virtio::Virtqueue::Desc::Flags::raw
l4_uint16_t raw
raw flags value of a virtio descriptor.
Definition virtqueue:94
L4virtio::Virtqueue::Desc::Flags::indirect
constexpr indirect_bfm_t::Val indirect() const
Get the indirect bits (2 to 2) of raw.
Definition virtqueue:105
L4virtio::Virtqueue::Used::Flags
flags for the used ring.
Definition virtqueue:180
L4virtio::Virtqueue::Used::Flags::no_notify
constexpr no_notify_bfm_t::Val no_notify() const
Get the no_notify bits (0 to 0) of raw.
Definition virtqueue:188
L4virtio::Virtqueue::Used::Flags::raw
l4_uint16_t raw
raw flags value as specified by virtio.
Definition virtqueue:181
L4virtio::Virtqueue::Used::Flags::Flags
Flags(l4_uint16_t v)
make Flags from raw value
Definition virtqueue:185
L4virtio::Virtqueue::Used_elem
Type of an element of the used ring.
Definition virtqueue:155
L4virtio::Virtqueue::Used_elem::id
l4_uint32_t id
descriptor index
Definition virtqueue:166
L4virtio::Virtqueue::Used_elem::len
l4_uint32_t len
length field
Definition virtqueue:167
L4virtio::Virtqueue::Used_elem::Used_elem
Used_elem(l4_uint16_t id, l4_uint32_t len)
Initialize a used ring element.
Definition virtqueue:165