L4Re – L4 Runtime Environment
virtqueue
1 // vi:ft=cpp
2 /*
3  * (c) 2014 Alexander Warg <warg@os.inf.tu-dresden.de>
4  *
5  * This file is part of TUD:OS and distributed under the terms of the
6  * GNU General Public License 2.
7  * Please see the COPYING-GPL-2 file for details.
8  *
9  * As a special exception, you may use this file as part of a free software
10  * library without restriction. Specifically, if other files instantiate
11  * templates or use macros or inline functions from this file, or you compile
12  * this file and link it with other files to produce an executable, this
13  * file does not by itself cause the resulting executable to be covered by
14  * the GNU General Public License. This exception does not however
15  * invalidate any other reasons why the executable file might be covered by
16  * the GNU General Public License.
17  */
18 
19 #include <l4/re/util/debug>
20 #include <l4/sys/types.h>
21 #include <l4/sys/err.h>
22 #include <l4/cxx/bitfield>
23 #include <l4/cxx/exceptions>
24 #include <cstdint>
25 
26 #pragma once
27 
28 namespace L4virtio {
29 
30 #ifdef __ARM_ARCH_7A__
31 static inline void wmb() { asm volatile ("dmb" : : : "memory"); }
32 static inline void rmb() { asm volatile ("dmb" : : : "memory"); }
33 #elif defined(__ARM_ARCH_8A)
34 static inline void wmb() { asm volatile ("dsb ishst" : : : "memory"); }
35 static inline void rmb() { asm volatile ("dsb ishld" : : : "memory"); }
36 #elif defined(__mips__)
37 static inline void wmb() { asm volatile ("sync" : : : "memory"); }
38 static inline void rmb() { asm volatile ("sync" : : : "memory"); }
39 #elif defined(__amd64__) || defined(__i386__) || defined(__i686__)
40 static inline void wmb() { asm volatile ("sfence" : : : "memory"); }
41 static inline void rmb() { asm volatile ("lfence" : : : "memory"); }
42 #else
43 #warning Missing proper memory write barrier
44 static inline void wmb() { asm volatile ("" : : : "memory"); }
45 static inline void rmb() { asm volatile ("" : : : "memory"); }
46 #endif
47 
48 
55 template< typename T >
56 class Ptr
57 {
58 public:
61 
62  Ptr() = default;
63 
65  Ptr(Invalid_type) : _p(~0ULL) {}
66 
68  explicit Ptr(l4_uint64_t vm_addr) : _p(vm_addr) {}
69 
71  l4_uint64_t get() const { return _p; }
72 
74  bool is_valid() const { return _p != ~0ULL; }
75 
76 private:
77  l4_uint64_t _p;
78 };
79 
80 
89 class Virtqueue
90 {
91 public:
95  class Desc
96  {
97  public:
101  struct Flags
102  {
104  Flags() = default;
105 
107  explicit Flags(l4_uint16_t v) : raw(v) {}
108 
110  CXX_BITFIELD_MEMBER( 0, 0, next, raw);
112  CXX_BITFIELD_MEMBER( 1, 1, write, raw);
114  CXX_BITFIELD_MEMBER( 2, 2, indirect, raw);
115  };
116 
121 
125  void dump(unsigned idx) const
126  {
127  L4Re::Util::Dbg().printf("D[%04x]: %08llx (%x) f=%04x n=%04x\n",
128  idx, addr.get(),
129  len, (unsigned)flags.raw, (unsigned)next);
130  }
131  };
132 
136  class Avail
137  {
138  public:
142  struct Flags
143  {
145  Flags() = default;
146 
148  explicit Flags(l4_uint16_t v) : raw(v) {}
149 
151  CXX_BITFIELD_MEMBER( 0, 0, no_irq, raw);
152  };
153 
157  };
158 
162  struct Used_elem
163  {
164  Used_elem() = default;
165 
176  };
177 
181  class Used
182  {
183  public:
187  struct Flags
188  {
190  Flags() = default;
191 
193  explicit Flags(l4_uint16_t v) : raw(v) {}
194 
196  CXX_BITFIELD_MEMBER( 0, 0, no_notify, raw);
197  };
198 
202  };
203 
204 protected:
208 
211 
217 
221  Virtqueue() : _desc(0), _idx_mask(0) {}
222  Virtqueue(Virtqueue const &) = delete;
223 
224 public:
230  void disable()
231  { _desc = 0; }
232 
236  enum
237  {
238  Desc_align = 4, //< Alignment of the descriptor table.
239  Avail_align = 1, //< Alignment of the available ring.
240  Used_align = 2, //< Alignment of the used ring.
241  };
242 
251  static unsigned long total_size(unsigned num)
252  {
253  static_assert(Desc_align >= Avail_align,
254  "virtqueue alignment assumptions broken");
255  return l4_round_size(desc_size(num) + avail_size(num), Used_align)
256  + used_size(num);
257  }
258 
267  static unsigned long desc_size(unsigned num)
268  { return num * 16; }
269 
275  static unsigned long desc_align()
276  { return Desc_align; }
277 
285  static unsigned long avail_size(unsigned num)
286  { return 2 * num + 6; }
287 
293  static unsigned long avail_align()
294  { return Avail_align; }
295 
304  static unsigned long used_size(unsigned num)
305  { return 8 * num + 6; }
306 
312  static unsigned long used_align()
313  { return Used_align; }
314 
320  unsigned long total_size() const
321  {
322  return ((char *) _used - (char *) _desc)
323  + used_size(num());
324  }
325 
329  unsigned long avail_offset() const
330  { return (char const *)_avail - (char const *)_desc; }
331 
335  unsigned long used_offset() const
336  { return (char const *)_used - (char const *)_desc; }
337 
355  void setup(unsigned num, void *desc, void *avail, void *used)
356  {
357  if (num > 0x10000)
358  throw L4::Runtime_error(-L4_EINVAL, "Queue too large.");
359 
360  _idx_mask = num - 1;
361  _desc = (Desc*)desc;
362  _avail = (Avail*)avail;
363  _used = (Used*)used;
364 
365  _current_avail = 0;
366 
367  L4Re::Util::Dbg().printf("VQ[%p]: num=%d d:%p a:%p u:%p\n",
368  this, num, _desc, _avail, _used);
369  }
370 
384  void setup_simple(unsigned num, void *ring)
385  {
386  l4_addr_t desc = reinterpret_cast<l4_addr_t>(ring);
387  l4_addr_t avail = l4_round_size(desc + desc_size(num), Avail_align);
388  l4_addr_t used = l4_round_size(avail + avail_size(num), Used_align);
389  setup(num, (void *)desc, (void *)avail, (void *)used);
390  }
391 
397  void dump(Desc const *d) const
398  { d->dump(d - _desc); }
399 
405  bool ready() const
406  { return L4_LIKELY(_desc != 0); }
407 
409  unsigned num() const
410  { return _idx_mask + 1; }
411 
419  bool no_notify_guest() const
420  {
421  return _avail->flags.no_irq();
422  }
423 
431  bool no_notify_host() const
432  {
433  return _used->flags.no_notify();
434  }
435 
441  void no_notify_host(bool value)
442  {
443  _used->flags.no_notify() = value;
444  }
445 
454  l4_uint16_t get_avail_idx() const { return _avail->idx; }
455 
462 
463 };
464 
465 namespace Driver {
466 
476 {
477 private:
479  l4_uint16_t _next_free;
480 
481 public:
482  enum End_of_queue
483  {
484  // Indicates the end of the queue.
485  Eoq = 0xFFFF
486  };
487 
488  Virtqueue() : _next_free(Eoq) {}
489 
499  void initialize_rings(unsigned num)
500  {
501  _used->idx = 0;
502  _avail->idx = 0;
503 
504  // setup the freelist
505  for (l4_uint16_t d = 0; d < num - 1; ++d)
506  _desc[d].next = d + 1;
507  _desc[num - 1].next = Eoq;
508  _next_free = 0;
509  }
510 
527  void init_queue(unsigned num, void *desc, void *avail, void *used)
528  {
529  setup(num, desc, avail, used);
531  }
532 
542  void init_queue(unsigned num, void *base)
543  {
544  setup_simple(num, base);
546  }
547 
548 
564  {
565  l4_uint16_t idx = _next_free;
566  if (idx == Eoq)
567  return Eoq;
568 
569  _next_free = _desc[idx].next;
570 
571  return idx;
572  }
573 
580  {
581  if (descno > _idx_mask)
582  throw L4::Bounds_error();
583 
584  _avail->ring[_avail->idx & _idx_mask] = descno; // _avail->idx expected to wrap
585  wmb();
586  ++_avail->idx;
587  }
588 
596  {
597  if (descno > _idx_mask)
598  throw L4::Bounds_error();
599 
600  return _desc[descno];
601  }
602 
615  {
616  if (_current_avail == _used->idx)
617  return Eoq;
618 
619  auto elem = _used->ring[_current_avail++ & _idx_mask];
620 
621  if (len)
622  *len = elem.len;
623 
624  return elem.id;
625  }
626 
637  {
638  if (head > _idx_mask || tail > _idx_mask)
639  throw L4::Bounds_error();
640 
641  _desc[tail].next = _next_free;
642  _next_free = head;
643  }
644 };
645 
646 }
647 } // namespace L4virtio
Access out of bounds.
Definition: exceptions:290
Exception for an abstract runtime error.
Definition: exceptions:140
Driver-side implementation of a Virtqueue.
Definition: virtqueue:476
void free_descriptor(l4_uint16_t head, l4_uint16_t tail)
Free a chained list of descriptors in the descriptor queue.
Definition: virtqueue:636
void enqueue_descriptor(l4_uint16_t descno)
Enqueue a descriptor in the available ring.
Definition: virtqueue:579
void init_queue(unsigned num, void *base)
Initialize this virtqueue.
Definition: virtqueue:542
Desc & desc(l4_uint16_t descno)
Return a reference to a descriptor in the descriptor table.
Definition: virtqueue:595
l4_uint16_t alloc_descriptor()
Allocate and return an unused descriptor from the descriptor table.
Definition: virtqueue:563
l4_uint16_t find_next_used(l4_uint32_t *len=nullptr)
Return the next finished block.
Definition: virtqueue:614
void init_queue(unsigned num, void *desc, void *avail, void *used)
Initialize this virtqueue.
Definition: virtqueue:527
void initialize_rings(unsigned num)
Initialize the descriptor table and the index structures of this queue.
Definition: virtqueue:499
Pointer used in virtio descriptors.
Definition: virtqueue:57
Ptr(l4_uint64_t vm_addr)
Make a Ptr from a raw 64bit address.
Definition: virtqueue:68
l4_uint64_t get() const
Definition: virtqueue:71
Invalid_type
Type for making an invalid (NULL) Ptr.
Definition: virtqueue:60
@ Invalid
Use to set a Ptr to invalid (NULL)
Definition: virtqueue:60
bool is_valid() const
Definition: virtqueue:74
Ptr(Invalid_type)
Make and invalid Ptr.
Definition: virtqueue:65
Type of available ring, this is read-only for the host.
Definition: virtqueue:137
l4_uint16_t ring[]
array of available descriptor indexes.
Definition: virtqueue:156
Flags flags
flags of available ring
Definition: virtqueue:154
l4_uint16_t idx
available index written by guest
Definition: virtqueue:155
Descriptor in the descriptor table.
Definition: virtqueue:96
l4_uint16_t next
Index of the next chained descriptor.
Definition: virtqueue:120
l4_uint32_t len
Length of described buffer.
Definition: virtqueue:118
Flags flags
Descriptor flags.
Definition: virtqueue:119
void dump(unsigned idx) const
Dump a single descriptor.
Definition: virtqueue:125
Ptr< void > addr
Address stored in descriptor.
Definition: virtqueue:117
Used_elem ring[]
array of used descriptors.
Definition: virtqueue:201
l4_uint16_t idx
index of the last entry in the ring.
Definition: virtqueue:200
Flags flags
flags of the used ring.
Definition: virtqueue:199
Low-level Virtqueue.
Definition: virtqueue:90
void no_notify_host(bool value)
Set the no-notify flag for this queue.
Definition: virtqueue:441
void disable()
Completely disable the queue.
Definition: virtqueue:230
void setup(unsigned num, void *desc, void *avail, void *used)
Enable this queue.
Definition: virtqueue:355
static unsigned long desc_size(unsigned num)
Calculate the size of the descriptor table for num entries.
Definition: virtqueue:267
l4_uint16_t get_tail_avail_idx() const
Get tail-available index stored in local state (for debugging).
Definition: virtqueue:461
Used * _used
pointer to used ring.
Definition: virtqueue:207
bool no_notify_guest() const
Get the no IRQ flag of this queue.
Definition: virtqueue:419
static unsigned long avail_align()
Get the alignment in zero LSBs needed for the available ring.
Definition: virtqueue:293
void dump(Desc const *d) const
Dump descriptors for this queue.
Definition: virtqueue:397
void setup_simple(unsigned num, void *ring)
Enable this queue.
Definition: virtqueue:384
unsigned long avail_offset() const
Get the offset of the available ring from the descriptor table.
Definition: virtqueue:329
static unsigned long used_align()
Get the alignment in zero LSBs needed for the used ring.
Definition: virtqueue:312
static unsigned long total_size(unsigned num)
Calculate the total size for a virtqueue of the given dimensions.
Definition: virtqueue:251
static unsigned long desc_align()
Get the alignment in zero LSBs needed for the descriptor table.
Definition: virtqueue:275
static unsigned long used_size(unsigned num)
Calculate the size of the used ring for num entries.
Definition: virtqueue:304
unsigned long used_offset() const
Get the offset of the used ring from the descriptor table.
Definition: virtqueue:335
static unsigned long avail_size(unsigned num)
Calculate the size of the available ring for num entries.
Definition: virtqueue:285
unsigned long total_size() const
Calculate the total size of this virtqueue.
Definition: virtqueue:320
Virtqueue()
Create a disabled virtqueue.
Definition: virtqueue:221
bool ready() const
Test if this queue is in working state.
Definition: virtqueue:405
l4_uint16_t _idx_mask
mask used for indexing into the descriptor table and the rings.
Definition: virtqueue:216
Desc * _desc
pointer to descriptor table, NULL if queue is off.
Definition: virtqueue:205
l4_uint16_t get_avail_idx() const
Get available index from available ring (for debugging).
Definition: virtqueue:454
bool no_notify_host() const
Get the no notify flag of this queue.
Definition: virtqueue:431
Avail * _avail
pointer to available ring.
Definition: virtqueue:206
l4_uint16_t _current_avail
The life counter for the queue.
Definition: virtqueue:210
unsigned num() const
Definition: virtqueue:409
#define L4_LIKELY(x)
Expression is likely to execute.
Definition: compiler.h:237
Error codes.
Base exceptions.
unsigned long l4_addr_t
Address type.
Definition: l4int.h:45
unsigned int l4_uint32_t
Unsigned 32bit value.
Definition: l4int.h:40
unsigned short int l4_uint16_t
Unsigned 16bit value.
Definition: l4int.h:38
unsigned long long l4_uint64_t
Unsigned 64bit value.
Definition: l4int.h:42
@ L4_EINVAL
Invalid argument.
Definition: err.h:56
l4_addr_t l4_round_size(l4_addr_t value, unsigned char bits) L4_NOTHROW
Round value up to the next alignment with bits size.
Definition: consts.h:400
Common L4 ABI Data Types.
L4-VIRTIO Transport C++ API.
Definition: virtio-block:29
Flags of the available ring.
Definition: virtqueue:143
no_irq_bfm_t::Val no_irq() const
Get the no_irq bits ( 0 to 0 ) of raw.
Definition: virtqueue:151
Flags(l4_uint16_t v)
Make Flags from the raw value.
Definition: virtqueue:148
l4_uint16_t raw
raw 16bit flags value of the available ring.
Definition: virtqueue:144
Type for descriptor flags.
Definition: virtqueue:102
write_bfm_t::Val write() const
Get the write bits ( 1 to 1 ) of raw.
Definition: virtqueue:112
next_bfm_t::Val next() const
Get the next bits ( 0 to 0 ) of raw.
Definition: virtqueue:110
indirect_bfm_t::Val indirect() const
Get the indirect bits ( 2 to 2 ) of raw.
Definition: virtqueue:114
Flags(l4_uint16_t v)
Make Flags from raw 16bit value.
Definition: virtqueue:107
l4_uint16_t raw
raw flags value of a virtio descriptor.
Definition: virtqueue:103
flags for the used ring.
Definition: virtqueue:188
l4_uint16_t raw
raw flags value as specified by virtio.
Definition: virtqueue:189
no_notify_bfm_t::Val no_notify() const
Get the no_notify bits ( 0 to 0 ) of raw.
Definition: virtqueue:196
Flags(l4_uint16_t v)
make Flags from raw value
Definition: virtqueue:193
Type of an element of the used ring.
Definition: virtqueue:163
l4_uint32_t id
descriptor index
Definition: virtqueue:174
l4_uint32_t len
length field
Definition: virtqueue:175
Used_elem(l4_uint16_t id, l4_uint32_t len)
Initialize a used ring element.
Definition: virtqueue:173