thread.h

Go to the documentation of this file.

// AUTOMATICALLY GENERATED -- DO NOT EDIT!         -*- c++ -*-

#ifndef thread_h
#define thread_h

#include <setjmp.h>             // typedef jmp_buf

#include "l4_types.h"

#include "activation.h"
#include "config.h"
#include "deadline_timeout.h"
#include "mem_layout.h"
#include "preemption.h"
#include "receiver.h"
#include "sender.h"
#include "space.h"              // Space_index
#include "thread_lock.h"

#include "trap_state.h"

//
// INTERFACE definition follows 
//


class Irq_alloc;
class Return_frame;
class Syscall_frame;
class Task;

class Idt_entry;

class Trap_state;
class Sys_ex_regs_frame;

class Sys_ipc_frame;

class Thread : public Receiver, public Sender
{
  friend class Jdb;
  friend class Jdb_bt;
  friend class Jdb_tcb;
  friend class Jdb_thread_list;
  friend class Jdb_list_threads;
  friend class Jdb_list_timeouts;
  friend class Jdb_tbuf_show;

public:

  typedef void (Utcb_copy_func)(Thread *sender, Thread *receiver);

  Thread (Task* task, Global_id id,
          unsigned short init_prio, unsigned short mcp);

  void sys_ipc();
  void sys_fpage_unmap();
  void sys_thread_switch();
  void sys_thread_schedule();
  void sys_task_new();
  void sys_id_nearest();
  void sys_thread_ex_regs();

  bool handle_page_fault (Address pfa, Mword error, Mword pc);

  Task_num Thread::d_taskno();

  LThread_num Thread::d_threadno();

private:
  Thread(const Thread&);        
  void *operator new(size_t);   

  bool handle_sigma0_page_fault (Address pfa);
  bool handle_smas_page_fault (Address pfa, Mword error, Ipc_err &ipc_code);

  void kill_small_space();

  Mword small_space();

  void set_small_space (Mword nr);

  static void user_invoke();

  void rcv_startup_msg();

  bool Thread::associate_irq(Irq_alloc *irq);
  void Thread::disassociate_irq(Irq_alloc *irq);

public:
  static Thread * Thread::lookup (L4_uid id, Space *s);

  static Mword pagein_tcb_request(Address pc);
  Mword is_tcb_mapped() const;

protected:
  // implementation details follow...

  // DATA

  // Preemption IPC sender role
  Preemption _preemption;

  // Deadline Timeout
  Deadline_timeout _deadline_timeout;

  // Activation IPC sender role
  Activation _activation;

  // Another critical TCB cache line:
  Task*        _task;
  Thread_lock  _thread_lock;

  // More ipc state
  Thread *_pager, *_ext_preempter;
  Thread *present_next, *present_prev;
//  Irq_alloc *_irq;

  // long ipc state
  L4_rcv_desc _target_desc;     // ipc buffer in receiver's address space
  unsigned _pagein_status_code;

  Address _vm_window0, _vm_window1; // data windows for the
                                // IPC partner's address space (for long IPC)
  jmp_buf *_recover_jmpbuf;     // setjmp buffer for page-fault recovery
  L4_timeout _pf_timeout;       // page-fault timeout specified by partner

  // debugging stuff
  Address _last_pf_address;
  unsigned _last_pf_error_code;

  unsigned _magic;
  static const unsigned magic = 0xf001c001;

  // Constructor
  Thread (Task* task, L4_uid id);

  // Thread killer
  void kill_all();
private:

protected:
  Irq_alloc *_irq;
private:

private:
  static Mword const exception_cs();

protected:
  Idt_entry *_idt;
  Unsigned16 _idt_limit;

  static Trap_state::Handler nested_trap_handler FIASCO_FASTCALL;
private:

private:
  // small address space stuff
  bool        handle_smas_gp_fault ();
  static int  (*int3_handler)(Trap_state*);
private:

public:
  inline bool ipc_short_cut();

private:
  Mword ipc_send_regs (Thread* receiver,
                       Sys_ipc_frame const *sender_regs);

  void page_fault_msg (Sys_ipc_frame &r, Address fault_address, 
                       Mword fault_ip, Mword err);

  void commit_ipc_success (Sys_ipc_frame *regs, Ipc_err err);

  void commit_ipc_failure (Sys_ipc_frame *regs, Ipc_err err);

  Ipc_err get_ipc_err (Sys_ipc_frame *regs);

  void set_source_local_id(Thread *receiver,  Sys_ipc_frame *dest_regs);
private:

private:
  Address remote_fault_addr (Address pfa);

  Mword update_ipc_window (Address pfa, Mword error_code, Address remote_pfa);
private:

private:
  static void handle_double_fault (void) asm ("thread_handle_double_fault");

public:
  static bool may_enter_jdb;

public:  
  inline void * operator new(size_t, L4_uid id);
  
  inline void operator delete(void *);
  
  virtual ~Thread();            // To be called in locked state.
  
  static inline Thread* lookup(Context* c);
  
  //
  // state requests/manipulation
  //
  
  inline Task * task() const;
  
  inline Thread_lock * thread_lock();
  
  inline Preemption * preemption();
  
  inline void handle_timer_interrupt();
  
  static inline Thread * lookup(Global_id id);
  
  void halt();
  
  static void halt_current();
  
  static inline Thread * lookup_first_thread(unsigned space);
  
  inline Space_index space_index() const;
  
  inline Space_index chief_index() const;
  
  bool kill_task(Space_index subtask);
  
  inline unsigned nest() const;
  
  inline bool has_privileged_iopl();
  
  inline void unset_utcb_ptr();
  
  static void privilege_initialization();
  
  bool initialize(Address ip, Address sp, Thread* pager, Receiver* preempter, Address *o_ip = 0, Address *o_sp = 0, Thread* *o_pager = 0, Receiver* *o_preempter = 0, Address *o_flags = 0, bool no_cancel = 0, bool alien = 0);
  
  int handle_slow_trap(Trap_state *ts);
  
  inline bool raise_exception(Trap_state *ts, Address handler);
  
  inline int snd_exception(Trap_state *) const;
  
  /*
   * Handle FPU trap for this context. Assumes disabled interrupts
   */
  inline void handle_fpu_trap();
  
  static inline void set_int3_handler(int (*handler)(Trap_state *ts));
  
  virtual void ipc_receiver_ready();
  
  Ipc_err ipc_continue(Ipc_err ipc_code);
  
  inline Thread * next_present() const;
  
  static inline int log_page_fault();
  
  inline void sys_ipc_log();
  
  inline void sys_ipc_trace();
  
  inline void sys_fpage_unmap_log();
  
  // Note that we don't want to check for Thread_invalid since we don't want
  // to raise page faults from inside the kernel debugger
  inline int is_mapped();
  
  // check if thread is valid (i.e. valid address, thread mapped)
  int is_valid();
  
  void print_snd_partner(int task_format=0);
  
  // Be robust if partner is invalid
  void print_partner(int task_format=0);
  
  // Be robust if this object is invalid
  void print_uid(int task_format=0);
  
  void print_state_long(unsigned cut_on_len = 0);
  
  static inline Task_num get_task(Global_id id);

protected:  
  inline void reset_ipc_window();
  
  inline void setup_ipc_window(unsigned win, Address address);
  
  inline bool in_present_list();
  
  inline void present_enqueue(Thread *sibling);
  
  inline void present_dequeue();

private:  
  static inline void user_invoke_generic();
  
  bool kill();
  
  inline void copy_utcb_to(Thread*);
  
  inline void enqueue_thread0_other_task();
  
  inline int is_privileged_for_debug(Trap_state * /*ts*/);
  
  static void print_page_fault_error(Mword e);
  
  inline bool get_ioport(Address /*eip*/, Trap_state * /*ts*/,
  unsigned * /*port*/, unsigned * /*size*/);
  
  inline void enqueue_thread_other_task();
  
  inline void setup_lipc_utcb();
  
  inline void setup_exception_ipc();
  
  inline void setup_utcb_kernel_addr();
  
  inline void arch_init();
  
  static int handle_int3(Trap_state *ts);
  
  static int call_nested_trap_handler(Trap_state *ts);
  
  inline int check_trap13_kernel(Trap_state *ts, bool from_user);
  
  inline void check_f00f_bug(Trap_state *ts);
  
  inline bool handle_io_page_fault(Trap_state *ts, Address eip, bool from_user);
  
  inline bool handle_sysenter_trap(Trap_state *ts, Address eip, bool from_user);
  
  inline bool trap_is_privileged(Trap_state *);
  
  inline void do_wrmsr_in_kernel(Trap_state *ts);
  
  inline void do_rdmsr_in_kernel(Trap_state *ts);
  
  inline int handle_not_nested_trap(Trap_state *ts);
  
  inline bool handle_lldt(Trap_state *);
  
  /*
   * Return current timesharing parameters
   */
  inline void get_timesharing_param(L4_sched_param *param, L4_uid *preempter, L4_uid *ipc_partner);
  
  /*
   * Set scheduling parameters for timeslice with id 'id'
   */
  inline Mword set_schedule_param(L4_sched_param param, unsigned short const id);
  
  /*
   * Set preempter unless "invalid"
   */
  inline void set_preempter(L4_uid const preempter);
  
  /*
   * Add a realtime timeslice at the end of the list
   */
  inline Mword set_realtime_param(L4_sched_param param);
  
  /*
   * Remove all realtime timeslices
   */
  inline Mword remove_realtime_param();
  
  Mword begin_periodic(Unsigned64 clock, Mword const type);
  
  Mword end_periodic();
  
  inline void reset_nest(L4_uid id);
  
  inline void inc_nest(L4_uid id);
  
  inline void update_nest(L4_uid id);
  
  inline int handle_inter_task_ex_regs(Sys_ex_regs_frame *, L4_uid *, Thread **, Task **, Thread **);
  
  static inline Unsigned64 round_quantum(Unsigned64 quantum);
  
  inline Unsigned64 snd_timeout(L4_timeout t, Sys_ipc_frame const * regs);
  
  inline Unsigned64 rcv_timeout(L4_timeout t, Sys_ipc_frame const *regs);
  
  Ipc_err do_send(Thread *partner, L4_timeout t, Sys_ipc_frame *regs);
  
  inline void prepare_receive(Sender *partner, Sys_ipc_frame *regs);
  
  Ipc_err do_receive(Sender *sender, L4_timeout t, Sys_ipc_frame *regs);
  
  Ipc_err handle_page_fault_pager(Address pfa, Mword error_code);
  
  inline void unlock_receiver(Receiver *receiver, const Sys_ipc_frame*);
  
  Ipc_err handle_ipc_page_fault(Address pfa, Mword error_code);
  
  Ipc_err ipc_snd_fpage(Thread* receiver, L4_fpage from_fpage, L4_fpage to_fpage, Address offset, bool grant, bool finish, bool dont_switch);
  
  inline void wake_receiver(Thread *receiver);
  
  Ipc_err ipc_finish(Thread *receiver, Ipc_err new_state, bool dont_switch);
  
  Ipc_err ipc_pagein_request(Receiver* receiver, Address address, Mword error_code);
  
  inline bool invalid_ipc_buffer(void const *a);
  
  Ipc_err do_send_long(Thread *partner, Sys_ipc_frame *i_regs);
  
  void page_fault_log(Address pfa, unsigned error_code, unsigned eip);
};

// ----------------------------------------------------------------------------

/*
 * Fiasco Thread Code
 * Shared between UX and native IA32.
 */


extern void (*syscall_table[])();

class Pf_msg_utcb_saver
{
public:
  Pf_msg_utcb_saver (const Utcb *u);
  void restore (Utcb *u);
};

inline Thread* current_thread();

extern "C" FIASCO_FASTCALL int thread_page_fault(Address pfa, Mword error_code, Address ip, Mword flags, Return_frame *regs);

extern "C" FIASCO_FASTCALL int thread_handle_trap(Trap_state *ts);

// We are entering with disabled interrupts!
extern "C" FIASCO_FASTCALL void thread_timer_interrupt(Address ip);

extern "C" void thread_timer_interrupt_stop(void);

extern "C" void thread_timer_interrupt_slow(void);

// The "FPU not available" trap entry point
extern "C" void thread_handle_fputrap();

extern "C" void sys_thread_switch_wrapper();

// these wrappers must come last in the source so that the real sys-call 
// implementations can be inlined by g++
extern "C" void sys_fpage_unmap_wrapper();

extern "C" void sys_id_nearest_wrapper();

extern "C" void sys_thread_ex_regs_wrapper();

extern "C" void sys_task_new_wrapper();

extern "C" void sys_thread_schedule_wrapper();

extern "C" void sys_ipc_wrapper();

extern "C" void ipc_short_cut_wrapper();

extern "C" void sys_ipc_log_wrapper(void);

extern "C" void sys_ipc_trace_wrapper(void);

extern "C" void sys_fpage_unmap_log_wrapper(void);

//
// IMPLEMENTATION includes follow (for use by inline functions)
//

#include "fpu_alloc.h"
#include "timeout.h"

#include "fpu_state.h"
#include "gdt.h"
#include "entry_frame.h"
#include "task.h"

#include "mod32.h"
#include "map_util.h"
#include "space.h"
#include "lock_guard.h"
#include "kdb_ke.h"
#include "logdefs.h"
#include "regdefs.h"
#include "std_macros.h"
#include "vmem_alloc.h"
#include "warn.h"
#include "jdb_trace.h"
#include "cpu_lock.h"
#include "config.h"
#include "kmem.h"
#include "mem_layout.h"

#include <cstdio>
#include "processor.h"
#include "trap_state.h"

//
// IMPLEMENTATION of inline functions (and needed classes)
//



inline void *
Thread::operator new(size_t, L4_uid id)
{
  // Allocate TCB in TCB space.  Actually, do not allocate anything,
  // just return the address.  Allocation happens on the fly in
  // Thread::handle_page_fault().
  return static_cast <void*>(lookup (id));
}


inline void
Thread::operator delete(void *)
{
  // XXX should check if all thread blocks on a given page are free
  // and deallocate (or mark free) the page if so.  this should be
  // easy to detect since normally all threads of a given task are
  // destroyed at once at task deletion time
}


inline Thread*
Thread::lookup(Context* c)
{
  return reinterpret_cast<Thread*>(c);
}


//
// state requests/manipulation
//

inline Task *
Thread::task() const
{
  return _task;
}


inline Thread_lock *
Thread::thread_lock()
{
  return &_thread_lock;
}



inline Preemption *
Thread::preemption()
{
  return &_preemption;
}



inline void
Thread::handle_timer_interrupt()
{
  // XXX: This assumes periodic timers (i.e. bogus in one-shot mode)
  if (!Config::fine_grained_cputime)
    consume_time (Config::scheduler_granularity);

  // Check if we need to reschedule due to timeouts or wakeups
  if (Timeout::do_timeouts() && !Context::schedule_in_progress())
    {
      schedule();
      assert (timeslice_timeout->is_set());     // Coma check
    }
}


inline Thread *
Thread::lookup(Global_id id)
{
  return reinterpret_cast <Thread *>
    (id.is_invalid() || id.gthread() >= Config::max_threads() ?
     0 : Mem_layout::Tcbs + (id.gthread() * Config::thread_block_size));
}


inline Thread *
Thread::lookup_first_thread(unsigned space)
{ return lookup (Global_id (space, 0)); }


inline Space_index
Thread::space_index() const
{ return Space_index (id().task()); }


inline Space_index
Thread::chief_index() const
{ return Space_index (id().chief()); }


inline unsigned
Thread::nest() const
{
  return id().nest();
}



inline bool
Thread::has_privileged_iopl()
{
  return false;
}



inline void
Thread::unset_utcb_ptr() {}



inline bool
Thread::raise_exception(Trap_state *ts, Address handler)
{
  Mword ip = ts->ip(), *sp = (Mword *) ts->sp();
  bool error_code = false, fault_addr = false;

  {
    Lock_guard <Thread_lock> guard (thread_lock());

    if (!(state() & Thread_ready) || (state() & Thread_cancel))
      return false;

    switch (ts->trapno)
      {
        case 0xe:
          fault_addr = true;
        case 0x8:
        case 0xa:
        case 0xb:
        case 0xc:
        case 0xd:
        case 0x11:
          error_code = true;
      }

    ts->esp -= sizeof (Mword) * (error_code ? 4 : 3);
    ts->eip  = handler;
  }

  space()->poke_user (sp - 1, (Mword)ts->flags());
  space()->poke_user (sp - 2, (Mword)exception_cs());
  space()->poke_user (sp - 3, (Mword)ip);

  if (error_code)
    space()->poke_user (sp - 4, (Mword)ts->err);
  if (fault_addr)
    space()->poke_user (sp - 5, (Mword)ts->cr2);

  /* reset single trap flag to mirror cpu correctly */
  if (ts->trapno == 1)
    ts->eflags &= ~EFLAGS_TF;

  return true;
}



inline int
Thread::snd_exception(Trap_state *) const
{
  return 0;
}


/*
 * Handle FPU trap for this context. Assumes disabled interrupts
 */

inline void
Thread::handle_fpu_trap()
{
  // If we own the FPU, we should never be getting an "FPU unavailable" trap
  assert (Fpu::owner() != this);

  // Enable the FPU before accessing it, otherwise recursive trap
  Fpu::enable();

  // Save the FPU state of the previous FPU owner (lazy) if applicable
  if (Fpu::owner()) {
    Fpu::owner()->state_change_dirty (~Thread_fpu_owner, 0);
    Fpu::save_state (Fpu::owner()->fpu_state());
  }

  // Allocate FPU state slab if we didn't already have one
  if (!fpu_state()->state_buffer())
    Fpu_alloc::alloc_state (fpu_state());

  // Become FPU owner and restore own FPU state
  state_change_dirty (~0U, Thread_fpu_owner);
  Fpu::restore_state (fpu_state());

  Fpu::set_owner (this);
}



inline void
Thread::set_int3_handler(int (*handler)(Trap_state *ts))
{
  int3_handler = handler;
}



inline Thread *
Thread::next_present() const
{
  return present_next;
}



inline int
Thread::log_page_fault()
{
  return Jdb_pf_trace::log();
}

// Note that we don't want to check for Thread_invalid since we don't want
// to raise page faults from inside the kernel debugger

inline int
Thread::is_mapped()
{
  return Kmem::virt_to_phys((void*)this) != (Address)-1;
}



inline Task_num
Thread::get_task(Global_id id)
{ return id.task(); }


inline void             
Thread::reset_ipc_window()
{                
  _vm_window1 = _vm_window0 = (Address) -1;
}


inline void             
Thread::setup_ipc_window(unsigned win, Address address)
{                
  if (win == 0)
    {
      // Useless, because _vm_window0 contains the always the message buffer,
      // which we setup for every long ipc only once.
      // And because every thread switch flushes always the IPC window.
      // we need to update _vm_window0 every time.
      // This can be enabled, if we get an ipc window owner per space, 
      // like the fpu.
      // if (EXPECT_TRUE (_vm_window0 == address))
      //   return;
      _vm_window0 = address;
    }
  else
    {
      // Only makes sense if some strings are in the same 4mb region.
      if (EXPECT_FALSE (_vm_window1 == address))
        return;
      _vm_window1 = address;
    }

  // Pull in the mappings for the entire 8 MB window, by copying 2 PDE slots,
  // thereby replacing the old ones, based on the optimistic assumption that
  // the receiver's mappings are already set up appropriately. Note that this
  // does not prevent a pagefault on either of these mappings later on, e.g.
  // if the receiver's mapping is r/o here and needs to be r/w for Long-IPC.
  // Careful: for SMAS current_space() != space()
  current_space()->remote_update (Kmem::ipc_window (win),
                                  receiver()->space(), address, 2);
}


inline bool 
Thread::in_present_list()
{
  return present_next;
}


inline void
Thread::present_enqueue(Thread *sibling)
{
  Lock_guard<Cpu_lock> guard (&cpu_lock);

  if (! in_present_list())
    {
      present_next = sibling->present_next;
      present_prev = sibling;
      sibling->present_next = this;
      present_next->present_prev = this;
    }
}


inline void
Thread::present_dequeue()
{
  Lock_guard<Cpu_lock> guard (&cpu_lock);

  if (in_present_list())
    {
      present_prev->present_next = present_next;
      present_next->present_prev = present_prev;
      present_next /* = present_prev */ = 0;
    }
}



inline void
Thread::copy_utcb_to(Thread*)
{}


inline void
Thread::unlock_receiver(Receiver *receiver, const Sys_ipc_frame*)
{
  receiver->ipc_unlock();
}



inline void
Thread::wake_receiver(Thread *receiver)
{
  // If neither IPC partner is delayed, just update the receiver's state
  if (EXPECT_TRUE (!((state() | receiver->state()) & Thread_delayed_ipc)))
    {
      receiver->state_change (~Thread_ipc_mask, Thread_ready);
      return;
    }

  // Critical section if either IPC partner is delayed until its next period
  Lock_guard <Cpu_lock> guard (&cpu_lock);

  // Sender has no receive phase and deadline timeout already hit
  if ( (state() & (Thread_receiving |
                   Thread_delayed_deadline | Thread_delayed_ipc)) ==
      Thread_delayed_ipc)
    {
      state_change_dirty (~Thread_delayed_ipc, 0);
      switch_sched (sched_context()->next());
      _deadline_timeout.set (Timer::system_clock() + period());
    }

  // Receiver's deadline timeout already hit
  if ( (receiver->state() & (Thread_delayed_deadline |
                             Thread_delayed_ipc) ==   
                             Thread_delayed_ipc))      
    {
      receiver->state_change_dirty (~Thread_delayed_ipc, 0);   
      receiver->switch_sched (receiver->sched_context()->next());
      receiver->_deadline_timeout.set (Timer::system_clock() +
                                       receiver->period());
    }

  receiver->state_change_dirty (~(Thread_ipc_mask|Thread_delayed_ipc),
                                Thread_ready);
}



inline Thread *
Thread::lookup(L4_uid id, Space*)
{ return lookup ((Global_id) id); }



inline void
Thread::kill_all()
{
  kill_task (Space_index (Config::boot_id.task()));
}


inline Task_num Thread::d_taskno() { return space_index(); }

inline LThread_num Thread::d_threadno() { return id().lthread(); }



inline bool Thread::handle_smas_page_fault( Address, Mword, Ipc_err & )
{ return false; }



inline void Thread::kill_small_space(void)
{}



inline Mword Thread::small_space( void )
{ return 0; }



inline void Thread::set_small_space( Mword /*nr*/)
{}



inline void
Thread::rcv_startup_msg()
{}



inline Mword
Thread::is_tcb_mapped() const
{
  // Touch the state to page in the TCB. If we get a pagefault here,
  // the handler doesn't handle it but returns immediatly after
  // setting eax to 0xffffffff
  Mword pagefault_if_0;
  asm volatile ("xorl %%eax,%%eax               \n\t"
                "andl $0xffffffff, %%ss:(%%ecx) \n\t"
                "setnc %%al                     \n\t"
                : "=a" (pagefault_if_0) : "c" (&_state));
  return pagefault_if_0;
}


// 
// Public services
// 

inline bool
Thread::handle_sigma0_page_fault(Address pfa)
{
  size_t size;

  // Check if mapping a superpage doesn't exceed the size of physical memory
  if (Cpu::have_superpages() &&
      (pfa & Config::SUPERPAGE_MASK) + Config::SUPERPAGE_SIZE <
       Kip::k()->main_memory_high())
    {
      pfa &= Config::SUPERPAGE_MASK;
      size = Config::SUPERPAGE_SIZE;
    }
  else
    {
      pfa &= Config::PAGE_MASK;
      size = Config::PAGE_SIZE;
    }

  return space()->v_insert (pfa, pfa, size,
                            Space::Page_writable |
                            Space::Page_user_accessible)
                         != Space::Insert_err_nomem;
}



inline Mword
Thread::update_ipc_window(Address pfa, Address remote_pfa, Mword error_code)
{
  Space *remote = receiver()->space();

  // If the remote address space has a mapping for the page fault address and
  // it is writable or we didn't want to write to it, then we can simply copy
  // the mapping into our address space via space()->remote_update().
  // Otherwise return 0 to trigger a pagein_request upstream.

  if (EXPECT_TRUE (remote->mapped (remote_pfa, (error_code & PF_ERR_WRITE))))
    {
      //careful: for SMAS current_space() != space()
      current_space()->remote_update (pfa, remote, remote_pfa, 1);

      // It's OK if the PF occurs again: This can happen if we're
      // preempted after the call to remote_update() above.  (This code
      // corresponds to code in Thread::handle_page_fault() that
      // checks for double page faults.)

      if (Config::monitor_page_faults)
        _last_pf_address = (Address) -1;

      return 1;
    }

  return 0;
}



inline Mword const
Thread::exception_cs()
{
  return Gdt::gdt_code_user | Gdt::Selector_user;
}



inline bool
Thread::handle_smas_gp_fault()
{
  return false;
}


inline void
Thread::sys_fpage_unmap()
{
  Sys_unmap_frame *regs = sys_frame_cast<Sys_unmap_frame>(this->regs());

  fpage_unmap (space(), regs->fpage(), regs->self_unmap(), regs->downgrade());
}



inline Mword
Thread::ipc_send_regs(Thread* receiver, Sys_ipc_frame const *sender_regs)
{
  Mword ret;

  ret = receiver->ipc_try_lock (nonull_static_cast<Sender*>(current_thread()));

  if (EXPECT_FALSE (ret))
    {
      if ((Smword) ret < 0)
        return 0x80000000;      // transient error -- partner not ready

      return ret;
    }

  if (EXPECT_FALSE (state() & Thread_cancel))
    {
      receiver->ipc_unlock();
      return Ipc_err::Secanceled;
    }

  receiver->ipc_init(nonull_static_cast<Sender*>(current_thread()));

  if (!Config::deceit_bit_disables_switch && sender_regs->snd_desc().deceite())
    panic ("deceiving ipc");    // XXX unimplemented

  Sys_ipc_frame* dst_regs = receiver->rcv_regs();
  const L4_msgdope ret_dope(Sys_ipc_frame::num_reg_words(), 0);

  ret = 0;                              // status code: IPC successful
  dst_regs->msg_dope (ret_dope);        // status code: rcv'd 2 dw

  // dequeue from sender queue if enqueued
  sender_dequeue (receiver->sender_list());

  // Reset sender's timeout, if any.  Once we're here, we don't want
  // the timeout to reset the sender's ipc_in_progress state bit
  // (which it still needs for a subsequent receive operation).
  Timeout *t = _timeout;
  if (EXPECT_FALSE (t != 0))
    {
      t->reset();
      if (t->has_hit())         // too late?
        {
          // Fix: re-set the Thread_ipc_in_progress flag.  The
          // following algorithm makes sure we only set this flag if
          // Thread_cancel has not been set in-between.
          state_add (Thread_ipc_in_progress);
          if (state() & Thread_cancel)
            state_del (Thread_ipc_in_progress);
        }
    }

  // copy message register contents
  sender_regs->copy_msg (dst_regs);

  copy_utcb_to(receiver);

  // copy sender ID
  dst_regs->rcv_src (id());

  // disallow lipc of the receiver
  receiver->deny_lipc();

  // is this a fast (registers-only) receive operation?
  // the following operations can be short-cut here:
  // - short message sender to short message receiver
  // - (short) register message sender to long message receiver
  // the following operations can not be short-cut as we're not allowed
  // to touch user memory:
  // - long message sender to long message receiver (obviously)
  // - long message sender to short message receiver
  //   (the long message might actually fit into a short message)
  // - short-flexpage message sender to long message receiver
  //   (because the rcvr's msg buffer may contain a flexpage option)

  // There's a possible optimization for
  // short-flexpage-to-long-message-buffer transfers; see the top of
  // this file.

  if (EXPECT_TRUE (!sender_regs->snd_desc().is_long_ipc())
        // sender wants short register IPC
      || EXPECT_TRUE (sender_regs->snd_desc().msg() == 0
                       // sender wants short IPC
                       && (dst_regs->rcv_desc().is_register_ipc()
                            // receiver wants register IPC
                           || dst_regs->rcv_desc().rmap()
                               // receiver wants short flexpage
                         )
        ))
    {
      // short IPC!

      if (sender_regs->snd_desc().map()) // send short flexpage?
        {
          if (EXPECT_FALSE (! dst_regs->rcv_desc().rmap()) )
              // rcvr not expecting an fpage?
            {
              dst_regs->msg_dope_set_error(Ipc_err::Remsgcut);
              ret = Ipc_err::Semsgcut;
            }
          else
            {
              dst_regs->msg_dope_combine(
                fpage_map(space(),
                          L4_fpage(sender_regs->msg_word(1)),
                          receiver->space(), dst_regs->rcv_desc().fpage(),
                          sender_regs->msg_word(0) & Config::PAGE_MASK,
                          L4_fpage(sender_regs->msg_word(1)).grant()));

              if (dst_regs->msg_dope().rcv_map_failed())
                ret = Ipc_err::Semapfailed;
            }
        }

      /* set the source local-id */
      set_source_local_id(receiver, dst_regs);

      // (If we don't send a flexpage in our message, that's OK even
      // if the receiver expected one.  The receiver can tell from the
      // status code that he didn't receive one.)

      // IPC done -- reset states

      // After this point we are able to receive!
      // Make sure that we are not if there was a send error.
      state_del ((ret != 0 ? Thread_ipc_in_progress : 0) |
                  Thread_polling | Thread_send_in_progress);

      wake_receiver (receiver);

      if (!Config::deceit_bit_disables_switch ||
          !sender_regs->snd_desc().deceite())
          receiver->thread_lock()->set_switch_hint(SWITCH_ACTIVATE_LOCKEE);
    }
  else
    {
      // prepare long IPC

      // XXX check for cancel -- sender?  receiver?

      _target_desc = dst_regs->rcv_desc(); //ebp & ~1;

      // If the receive timeout has hit, ignore it.  We do this by
      // overwriting the Thread_ipc_in_process flag which the timeout
      // may have deleted -- but only if the Thread_cancel flag has
      // not been set in-between.

      receiver->reset_timeout();

      // set up page-fault timeouts
      L4_timeout t = sender_regs->timeout();
      if (t.snd_pfault() == 15) // send pfault timeout == 0 ms?
        receiver->_pf_timeout = L4_timeout(0,1,0,1,0,0);
      else
        {
          receiver->_pf_timeout = L4_timeout(1, t.snd_pfault(),
                                             1, t.snd_pfault(), 0, 0);
          // XXX should normalize timeout spec, but do_send/do_receive
          // can cope with non-normalized numbers.
        }

      t = dst_regs->timeout();
      if (t.rcv_pfault() == 15) // rcv pfault timeout == 0 ms?
        _pf_timeout = L4_timeout(0,1,0,1,0,0);
      else
        {
          _pf_timeout = L4_timeout(1, t.rcv_pfault(), 1, t.rcv_pfault(), 0, 0);
          // XXX should normalize timeout spec, but do_send/do_receive
          // can cope with non-normalized numbers.
        }

      // set up return code in case we're aborted
      dst_regs->msg_dope_set_error(Ipc_err::Reaborted);

      // switch receiver's state, and put it to sleep.
      // overwrite ipc_in_progress flag a timeout may have deleted --
      // see above
      receiver->state_change(~(Thread_receiving | Thread_busy | Thread_ready),
                             Thread_rcvlong_in_progress
                             | Thread_ipc_in_progress);
      if (receiver->state() & Thread_cancel)
          receiver->state_change(~Thread_ipc_in_progress, Thread_ready);

      // in the sender, send_in_progress must remain set
      state_del(Thread_polling);
    }

  unlock_receiver(receiver, sender_regs);
  return ret;
}



inline void
Thread::page_fault_msg(Sys_ipc_frame &r, Address fault_address,
                        Mword fault_eip, Mword error_code)
{
  r.set_msg_word (0, PF::addr_to_msgword0 (fault_address, error_code));
  r.set_msg_word (1, PF::pc_to_msgword1 (fault_eip, error_code));
  r.set_msg_word (2, 0); // nop in V2
  r.snd_desc (0);       // short msg
  r.rcv_desc(L4_rcv_desc::short_fpage(L4_fpage(0,0,L4_fpage::Whole_space,0)));
};



inline void
Thread::commit_ipc_success(Sys_ipc_frame *regs, Ipc_err err)
{
  regs->msg_dope (regs->msg_dope().raw_dope() | err.raw());
}



inline void
Thread::commit_ipc_failure(Sys_ipc_frame *regs, Ipc_err err)
{
  state_del (Thread_delayed_deadline | Thread_delayed_ipc);
  regs->msg_dope (0);
  commit_ipc_success (regs, err);
}



inline Ipc_err
Thread::get_ipc_err(Sys_ipc_frame *regs)
{
  return Ipc_err (regs->msg_dope().raw());
}

inline void
Thread::set_source_local_id(Thread *, Sys_ipc_frame *)
{}



inline Address
Thread::remote_fault_addr(Address pfa)
{
  return pfa < Kmem::ipc_window(1) ? pfa - Kmem::ipc_window(0) + _vm_window0
                                   : pfa - Kmem::ipc_window(1) + _vm_window1;
}



inline bool Thread::handle_page_fault(Address pfa, Mword error_code, Mword pc)
{
  if (Config::Log_kernel_page_faults && !PF::is_usermode_error(error_code))
    {
      printf("*KP[%lx,", pfa);
      print_page_fault_error(error_code);
      printf(",%lx]\n", pc);
    }
#if 0
  printf("Translation error ? %x\n"
         "  current space has mapping : %08x\n"
         "  Kernel space has mapping  : %08x\n",
         PF::is_translation_error(error_code),
         current_space()->lookup((void*)pfa),
         Space::kernel_space()->lookup((void*)pfa));
#endif

  if (Config::monitor_page_faults)
    {
      if (_last_pf_address == pfa && _last_pf_error_code == error_code)
        {
          if (!log_page_fault())
            printf("*P[%lx,%lx,%lx]", pfa, error_code & 0xffff, pc);
          putchar('\n');

          kdb_ke("PF happened twice");
        }

      _last_pf_address = pfa;
      _last_pf_error_code = error_code;

      // (See also corresponding code in Thread::handle_ipc_page_fault()
      //                          and in Thread::handle_slow_trap.)
    }

  CNT_PAGE_FAULT;

  // TODO: put this into a debug_page_fault_handler
  if (EXPECT_FALSE (log_page_fault()))
    page_fault_log (pfa, error_code, pc);

  Ipc_err ipc_code(0);

  // Check for page fault in user memory area
  if (EXPECT_TRUE (!Kmem::is_kmem_page_fault(pfa, error_code)))
    {
      // Make sure that we do not handle page faults that do not
      // belong to this thread.
      assert (space() == current_space());

      if (EXPECT_FALSE (space()->is_sigma0()))
        {
          // special case: sigma0 can map in anything from the kernel
          if(handle_sigma0_page_fault(pfa))
            return true;

          ipc_code.error (Ipc_err::Remapfailed);
          goto error;
        }

      // user mode page fault -- send pager request
      if (!(ipc_code = handle_page_fault_pager(pfa, error_code)).has_error())
        return true;

      goto error;
    }

  // Check for page fault in small address space
  else if (EXPECT_FALSE (Kmem::is_smas_page_fault(pfa)))
    {
      if (handle_smas_page_fault (pfa, error_code, ipc_code))
        return true;

      goto error;
    }

  // Check for page fault in kernel memory region caused by user mode
  else if (EXPECT_FALSE(PF::is_usermode_error(error_code)))
    return false;             // disallow access after mem_user_max

  // Check for page fault in IO bit map or in delimiter byte behind IO bitmap
  // assume it is caused by an input/output instruction and fall through to
  // handle_slow_trap
  else if (EXPECT_FALSE(Kmem::is_io_bitmap_page_fault(pfa)))
    return false;

  // We're in kernel code faulting on a kernel memory region

  // Check for page fault in IPC window. Mappings for this should never
  // be present in the global master page dir (case below), because the
  // IPC window mappings are always flushed on context switch.
  else if (EXPECT_TRUE (Kmem::is_ipc_page_fault(pfa, error_code)))
    {
      if (!(ipc_code = handle_ipc_page_fault(pfa, error_code)).has_error())
        return true;

      goto error;
    }

  // A page is not present but a mapping exists in the global page dir.
  // Update our page directory by copying from the master pdir
  // This is the only path that should be executed with interrupts
  // disabled if the page faulter also had interrupts disabled.   
  // thread_page_fault() takes care of that.
  else 
#ifdef CONFIG_ARM
  if (PF::is_translation_error(error_code) &&
      Space::kernel_space()->lookup((void*)pfa) != (Mword) -1)
#else
  if (PF::is_translation_error(error_code) &&
      Kmem::virt_to_phys (reinterpret_cast<void*>(pfa)) != (Mword) -1)
#endif
    {
      current_space()->kmem_update((void*)pfa);
      return true;
    }

  // Check for page fault in kernel's TCB area
  else if (Kmem::is_tcb_page_fault(pfa, error_code))
    {
      if (PF::is_translation_error(error_code))   // page not present
        {
          if (!PF::is_read_error(error_code))
            Proc::sti();

          if (!Vmem_alloc::page_alloc((void*)(pfa & Config::PAGE_MASK),
                                      PF::is_read_error(error_code) ?
                                      Vmem_alloc::ZERO_MAP:
                                      Vmem_alloc::ZERO_FILL)) 
            panic("can't alloc kernel page");
        }
      else
        { 
          // protection fault
          // this can only be because we have the zero page mapped
          Proc::sti();
          Vmem_alloc::page_free
            (reinterpret_cast<void*> (pfa & Config::PAGE_MASK));
          if (! Vmem_alloc::page_alloc((void*)(pfa & Config::PAGE_MASK),
                                       Vmem_alloc::ZERO_FILL))
            {
              // error could mean: someone else was faster allocating
              // a page there, or we just don't have any pages left; verify
#ifdef CONFIG_ARM
              if (Space::kernel_space()->lookup((void*)pfa) != (Mword) -1)
                panic("can't alloc kernel page");
#else
              if (Kmem::virt_to_phys(reinterpret_cast<void*>(pfa)) 
                  == Mword (-1))
                panic("can't alloc kernel page");
#endif

              // otherwise, there's a page mapped.  continue
            }
        }

      current_space()->kmem_update((void*)pfa);
      return true;
    }

  WARN("No page fault handler for 0x%lx, error 0x%lx, pc "L4_PTR_FMT"",
        pfa, error_code, pc);

  // An error occurred.  Our last chance to recover is an exception
  // handler a kernel function may have set.
 error:

  if (_recover_jmpbuf)
    longjmp (*_recover_jmpbuf, ipc_code.raw());

  return false;
}



inline Pf_msg_utcb_saver::Pf_msg_utcb_saver(const Utcb *)
{}



inline void
Pf_msg_utcb_saver::restore(Utcb *)
{}


inline Thread*
current_thread()
{
  return Thread::lookup(current());
}

#endif // thread_h

---