virtqueue

// vi:set ft=cpp: -*- Mode: C++ -*-
/* SPDX-License-Identifier: MIT */
/*
 * (c) 2014 Alexander Warg <warg@os.inf.tu-dresden.de>
 */
 
#include <l4/re/util/debug>
#include <l4/sys/types.h>
#include <l4/sys/err.h>
#include <l4/cxx/bitfield>
#include <l4/cxx/exceptions>
#include <cstdint>
 
#pragma once
 
namespace L4virtio {
 
#if defined(__ARM_ARCH) && __ARM_ARCH == 7
static inline void wmb() { asm volatile ("dmb ishst" : : : "memory"); }
static inline void rmb() { asm volatile ("dmb ish"   : : : "memory"); }
static inline void mb()  { asm volatile ("dmb ish"   : : : "memory"); }
#elif defined(__ARM_ARCH) && __ARM_ARCH >= 8
static inline void wmb() { asm volatile ("dmb ishst" : : : "memory"); }
static inline void rmb() { asm volatile ("dmb ishld" : : : "memory"); }
static inline void mb()  { asm volatile ("dmb ish"   : : : "memory"); }
#elif defined(__mips__)
static inline void wmb() { asm volatile ("sync" : : : "memory"); }
static inline void rmb() { asm volatile ("sync" : : : "memory"); }
static inline void mb()  { asm volatile ("sync" : : : "memory"); }
#elif defined(__amd64__) || defined(__i386__) || defined(__i686__)
static inline void wmb() { asm volatile ("sfence" : : : "memory"); }
static inline void rmb() { asm volatile ("lfence" : : : "memory"); }
static inline void mb()  { asm volatile ("mfence" : : : "memory"); }
#elif defined(__riscv)
static inline void wmb() { asm volatile ("fence ow, ow" : : : "memory"); }
static inline void rmb() { asm volatile ("fence ir, ir" : : : "memory"); }
static inline void mb()  { asm volatile ("fence iorw, iorw" : : : "memory"); }
#else
#warning Missing proper memory write barrier
static inline void wmb() { asm volatile ("" : : : "memory"); }
static inline void rmb() { asm volatile ("" : : : "memory"); }
static inline void mb()  { asm volatile ("" : : : "memory"); }
#endif
 

template< typename T >
class Ptr
{
public:
  enum Invalid_type { Invalid  };
 
  Ptr() = default;

  Ptr(Invalid_type) : _p(~0ULL) {}

  explicit Ptr(l4_uint64_t vm_addr) : _p(vm_addr) {}

  l4_uint64_t get() const { return _p; }

  bool is_valid() const { return _p != ~0ULL; }
 
private:
  l4_uint64_t _p;
};
 

class Virtqueue
{
public:
  class Desc
  {
  public:
    struct Flags
    {
      l4_uint16_t raw;  
      Flags() = default;

      explicit Flags(l4_uint16_t v) : raw(v) {}

      CXX_BITFIELD_MEMBER( 0,  0, next, raw);
      CXX_BITFIELD_MEMBER( 1,  1, write, raw);
      CXX_BITFIELD_MEMBER( 2,  2, indirect, raw);
    };
 
    Ptr<void> addr;   
    l4_uint32_t len;  
    Flags flags;      
    l4_uint16_t next; 

    void dump(unsigned idx, L4Re::Util::Dbg dbg = L4Re::Util::Dbg{}) const
    {
      dbg.printf("D[%04x]: %08llx (%x) f=%04x n=%04x\n",
                 idx, addr.get(), len, flags.raw, next);
    }
  };

  class Avail
  {
  public:
    struct Flags
    {
      l4_uint16_t raw; 
      Flags() = default;

      explicit Flags(l4_uint16_t v) : raw(v) {}

      CXX_BITFIELD_MEMBER( 0,  0, no_irq, raw);
    };
 
    Flags flags;         
    l4_uint16_t idx;     
    l4_uint16_t ring[];  
  };

  struct Used_elem
  {
    Used_elem() = default;

    Used_elem(l4_uint16_t id, l4_uint32_t len) : id(id), len(len) {}
    l4_uint32_t id;  
    l4_uint32_t len; 
  };

  class Used
  {
  public:
    struct Flags
    {
      l4_uint16_t raw;  
      Flags() = default;

      explicit Flags(l4_uint16_t v) : raw(v) {}

      CXX_BITFIELD_MEMBER( 0,  0, no_notify, raw);
    };
 
    Flags flags;       
    l4_uint16_t idx;   
    Used_elem ring[];  
  };
 
protected:
  Desc *_desc = nullptr; 
  Avail *_avail = nullptr; 
  Used *_used = nullptr;   

  l4_uint16_t _current_avail = 0;

  l4_uint16_t _idx_mask = 0;

  Virtqueue() = default;
 
  Virtqueue(Virtqueue const &) = delete;
  ~Virtqueue() = default;
 
public:
  void disable()
  { _desc = 0; }

  enum
  {
    Desc_align  = 4, //< Alignment of the descriptor table.
    Avail_align = 1, //< Alignment of the available ring.
    Used_align  = 2, //< Alignment of the used ring.
  };

  static unsigned long total_size(unsigned num)
  {
    static_assert(Desc_align >= Avail_align,
                  "virtqueue alignment assumptions broken");
    return l4_round_size(desc_size(num) + avail_size(num), Used_align)
           + used_size(num);
  }

  static unsigned long desc_size(unsigned num)
  { return num * 16; }

  static unsigned long desc_align()
  { return Desc_align; }

  static unsigned long avail_size(unsigned num)
  { return 2 * num + 6; }

  static unsigned long avail_align()
  { return Avail_align; }

  static unsigned long used_size(unsigned num)
  { return 8 * num + 6; }

  static unsigned long used_align()
  { return Used_align; }

  unsigned long total_size() const
  {
    return (reinterpret_cast<char *>(_used) - reinterpret_cast<char *>(_desc))
            + used_size(num());
  }

  unsigned long avail_offset() const
  { return reinterpret_cast<char *>(_avail) - reinterpret_cast<char *>(_desc); }

  unsigned long used_offset() const
  { return reinterpret_cast<char *>(_used) - reinterpret_cast<char *>(_desc); }

  void setup(unsigned num, void *desc, void *avail, void *used,
             L4Re::Util::Dbg msg_dev = L4Re::Util::Dbg{2})
  {
    if (num == 0 || (num & (num - 1)))
      throw L4::Runtime_error(-L4_EINVAL, "Queue size is not a power of two.");
    if (num > 0x10000)
      throw L4::Runtime_error(-L4_EINVAL, "Queue too large.");
 
    _idx_mask = num - 1;
    _desc = static_cast<Desc*>(desc);
    _avail = static_cast<Avail*>(avail);
    _used = static_cast<Used*>(used);
 
    _current_avail = 0;
 
    msg_dev.printf("VQ[%p]: num=%d d:%p a:%p u:%p\n",
                   this, num, _desc, _avail, _used);
  }

  void setup_simple(unsigned num, void *ring,
                    L4Re::Util::Dbg msg_dev = L4Re::Util::Dbg{2})
  {
    l4_addr_t desc = reinterpret_cast<l4_addr_t>(ring);
    l4_addr_t avail = l4_round_size(desc + desc_size(num), Avail_align);
    void *used = reinterpret_cast<void *>(
      l4_round_size(avail + avail_size(num), Used_align));
    setup(num, ring, reinterpret_cast<void *>(avail), used, msg_dev);
  }

  void dump(Desc const *d, L4Re::Util::Dbg dbg = L4Re::Util::Dbg{}) const
  { d->dump(d - _desc, dbg); }

  bool ready() const
  { return L4_LIKELY(_desc != 0); }

  unsigned num() const
  { return _idx_mask + 1; }

  bool no_notify_guest() const
  {
    mb(); // All queue updates must be visible before the flag can be read!
    return _avail->flags.no_irq();
  }

  bool no_notify_host() const
  {
    mb(); // All queue updates must be visible before the flag can be read!
    return _used->flags.no_notify();
  }

  void no_notify_host(bool value)
  {
    _used->flags.no_notify() = value;
  }

  l4_uint16_t get_avail_idx() const { return _avail->idx; }

  l4_uint16_t get_tail_avail_idx() const { return _current_avail; }
 
};
 
namespace Driver {

class Virtqueue : public L4virtio::Virtqueue
{
private:
  l4_uint16_t _next_free;
 
public:
  enum End_of_queue
  {
    // Indicates the end of the queue.
    Eoq = 0xFFFF
  };
 
  Virtqueue() : _next_free(Eoq) {}

  void initialize_rings(unsigned num)
  {
    _used->idx = 0;
    _avail->idx = 0;
 
    // setup the freelist
    for (l4_uint16_t d = 0; d < num - 1; ++d)
      _desc[d].next = d + 1;
    _desc[num - 1].next = Eoq;
    _next_free = 0;
  }

  void init_queue(unsigned num, void *desc, void *avail, void *used,
                  L4Re::Util::Dbg msg_dev = L4Re::Util::Dbg{2})
  {
    setup(num, desc, avail, used, msg_dev);
    initialize_rings(num);
  }

  void init_queue(unsigned num, void *base,
                  L4Re::Util::Dbg msg_dev = L4Re::Util::Dbg{2})
  {
    setup_simple(num, base, msg_dev);
    initialize_rings(num);
  }
 

  l4_uint16_t alloc_descriptor()
  {
    l4_uint16_t idx = _next_free;
    if (idx == Eoq)
      return Eoq;
 
    _next_free = _desc[idx].next;
 
    return idx;
  }

  void enqueue_descriptor(l4_uint16_t descno)
  {
    if (descno > _idx_mask)
      throw L4::Bounds_error();
 
    _avail->ring[_avail->idx & _idx_mask] = descno; // _avail->idx expected to wrap
    // Make the ring entry visible before publishing the new index, so the
    // device never sees an advanced index pointing at a stale entry. Pairs
    // with the rmb() on the consuming side.
    wmb();
    ++_avail->idx;
  }

  Desc &desc(l4_uint16_t descno)
  {
    if (descno > _idx_mask)
      throw L4::Bounds_error();
 
    return _desc[descno];
  }

  l4_uint16_t find_next_used(l4_uint32_t *len = nullptr)
  {
    if (_current_avail == _used->idx)
      return Eoq;
 
    // The device writes the used element (id and len) before it advances
    // _used->idx. Having observed the new idx above, this barrier ensures
    // we read the matching ring entry only afterwards and thus see the
    // device's writes, not stale contents. Pairs with the wmb() the device
    // side issues before publishing the index.
    rmb();
    auto elem = _used->ring[_current_avail++ & _idx_mask];
 
    if (len)
      *len = elem.len;
 
    return elem.id;
  }

  void free_descriptor(l4_uint16_t head, l4_uint16_t tail)
  {
    if (head > _idx_mask || tail > _idx_mask)
      throw L4::Bounds_error();
 
    _desc[tail].next = _next_free;
    _next_free = head;
  }
};
 
}
} // namespace L4virtio