l4re-base-25.08.0

src/l4/pkg/ipcbench/Control

@@ -0,0 +1,2 @@
+requires: stdlibs
+Maintainer: adam@l4re.org

src/l4/pkg/ipcbench/Makefile

@@ -0,0 +1,6 @@
+PKGDIR	= .
+L4DIR	?= $(PKGDIR)/../..
+
+TARGET := src conf
+
+include $(L4DIR)/mk/subdir.mk

src/l4/pkg/ipcbench/conf/Makefile

@@ -0,0 +1,8 @@
+PKGDIR ?= ..
+L4DIR  ?= $(PKGDIR)/../..
+
+PKGNAME = ipcbench
+SRC_ASSETS_NED = ipcbench.cfg
+SRC_ASSETS_MODLIST = modules.list
+
+include $(L4DIR)/mk/assets.mk

src/l4/pkg/ipcbench/conf/ipcbench.cfg

@@ -0,0 +1,30 @@
+-- SPDX-License-Identifier: MIT
+-- vim:set ft=lua:
+local L4 = require("L4");
+local l = L4.default_loader;
+
+print("IPC within one address space:");
+local e = l:start({ log = { "ipc:1AS" }}, "rom/ipcbench");
+e:wait();
+
+print("IPC between two address spaces:");
+local comm = l:new_channel();
+local s;
+s = l:start({ log = { "ipc:2AS" }, caps = { comm = comm:svr() }}, "rom/ipcbench_server");
+e = l:start({ log = { "ipc:2AS" }, caps = { comm = comm },     }, "rom/ipcbench_client");
+e:wait();
+s:kill();
+
+print("IPC within one address space, on all cores in parallel:");
+local e = l:start({ log = { "ipc:PL" }}, "rom/ipcbench_parallel");
+e:wait();
+
+print("Syscall benchmark:");
+e = l:start({ log = {"sc:single"}}, "rom/syscallbench");
+e:wait();
+
+print("Syscall benchmark, on all cores in parallel:");
+e = l:start({ log = { "sc:PL" }}, "rom/syscallbench_parallel");
+e:wait();
+
+print("Benchmarks done.");

src/l4/pkg/ipcbench/conf/modules.list

@@ -0,0 +1,11 @@
+entry ipcbench
+roottask moe rom/ipcbench.cfg
+module l4re
+module ned
+module ipcbench
+module ipcbench_client
+module ipcbench_server
+module ipcbench_parallel
+module syscallbench
+module syscallbench_parallel
+module ned/ipcbench/ipcbench.cfg

src/l4/pkg/ipcbench/src/Makefile

@@ -0,0 +1,15 @@
+PKGDIR        ?= ..
+L4DIR         ?= $(PKGDIR)/../..
+
+TARGET                 = ipcbench ipcbench_parallel \
+                         ipcbench_client ipcbench_server \
+                         syscallbench syscallbench_parallel
+REQUIRES_LIBS          = libpthread
+SRC_C_ipcbench         = ipcbench.c ipc_common.c
+SRC_C_ipcbench_parallel = ipcbench_parallel.c ipc_common.c
+SRC_C_ipcbench_client  = ipcclient.c
+SRC_C_ipcbench_server  = ipcserver.c
+SRC_C_syscallbench     = syscallbench.c ipc_common.c
+SRC_C_syscallbench_parallel = syscallbench_parallel.c ipc_common.c
+
+include $(L4DIR)/mk/prog.mk

src/l4/pkg/ipcbench/src/ipc_common.c

@@ -0,0 +1,195 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re syscall benchmark program */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <pthread-l4.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <l4/sys/ipc.h>
+#include <l4/sys/rcv_endpoint.h>
+#include <l4/sys/thread.h>
+#include <l4/re/env.h>
+
+#include "ipc_common.h"
+#include "measure.h"
+
+enum { Enable_return_checks = 1 };
+
+void check_pthr_err(int r, char const *msg)
+{
+  if (r != 0)
+    {
+      printf("error: %s: %s (%d)\n", msg, strerror(r), r);
+      exit(1);
+    }
+}
+
+void enumerate_cpus(void (*cb)(unsigned cpu, void *arg), void *arg)
+{
+  enum { Map_size = sizeof(l4_umword_t) * 8 };
+
+  l4_cap_idx_t sched = l4re_env()->scheduler;
+  l4_umword_t cpu_max = 0;
+  unsigned offset = 0;
+
+  do
+    {
+      l4_sched_cpu_set_t cpus = l4_sched_cpu_set(offset, 0, 0);
+      if (l4_error(l4_scheduler_info(sched, &cpu_max, &cpus)))
+        {
+          printf("Error enumerating CPUs!\n");
+          return;
+        }
+
+      for (unsigned i = 0; i < Map_size; i++)
+        if (cpus.map & (1UL << i))
+          cb(offset + i, arg);
+
+      offset += Map_size;
+    }
+  while (offset < cpu_max);
+}
+
+static unsigned start_sync_val;
+
+void wait_for_start()
+{
+  while (!start_sync_val)
+    asm volatile("" : : : "memory");
+}
+
+void start(void)
+{
+  start_sync_val = 1;
+}
+
+static unsigned completion_counter = 1;
+
+void set_completion_counter(unsigned val)
+{
+  completion_counter = val;
+}
+
+void finished(void)
+{
+  unsigned n = __atomic_sub_fetch(&completion_counter, 1, __ATOMIC_SEQ_CST);
+
+  if (n == 0)
+    start_sync_val = ~0u;
+  else
+    while (start_sync_val != ~0u)
+      asm volatile("" : : : "memory");
+}
+
+static void count_cpus_cb(unsigned cpu, void *arg)
+{
+  (void)cpu;
+  unsigned *num = (unsigned *)arg;
+  (*num)++;
+}
+
+unsigned count_cpus(void)
+{
+  unsigned num_cpus = 0;
+  enumerate_cpus(&count_cpus_cb, &num_cpus);
+  return num_cpus;
+}
+
+long run_thread(l4_cap_idx_t thread, unsigned cpu)
+{
+  l4_sched_param_t sp = l4_sched_param(2, 0);
+  sp.affinity = l4_sched_cpu_set(cpu, 0, 1);
+  return l4_error(l4_scheduler_run_thread(l4re_env()->scheduler, thread, &sp));
+}
+
+/* This thread is initiating IPC */
+void *fn_caller(void *cp)
+{
+  struct Caller_params *params = (struct Caller_params *)cp;
+
+  l4_utcb_t *utcb = l4_utcb();
+  l4_msgtag_t tag = l4_msgtag(0, 0, 0, 0);
+  l4_cap_idx_t responder_cap = params->responder_cap;
+
+  wait_for_start();
+
+  PREPARE();
+
+  l4_msgtag_t r;
+  UNIT_TYPE(start);
+  UNIT_TYPE(end);
+  TAKE_TIME(start);
+  SYNC();
+  for (int i = 0; i < Num_rounds; ++i)
+    {
+      r = l4_ipc_call(responder_cap, utcb, tag, L4_IPC_NEVER);
+      if (Enable_return_checks && l4_ipc_error(r, utcb))
+        printf("caller: ipc err\n");
+    }
+  SYNC();
+  TAKE_TIME(end);
+
+  finished();
+
+  PRINT_RESULT(params->cpu, start, end, "IPC", 2);
+
+  return NULL;
+}
+
+/* This thread is replying to IPC */
+void *fn_responder(void *ignore)
+{
+  (void)ignore;
+  l4_utcb_t *utcb = l4_utcb();
+  l4_umword_t label;
+
+  // Allow cancellation of thread at end of test
+  check_pthr_err(pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL),
+                 "pthread_setcanceltype");
+  check_pthr_err(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL),
+                 "pthread_setcancelstate");
+
+  l4_msgtag_t r = l4_ipc_wait(utcb, &label, L4_IPC_NEVER);
+  if (Enable_return_checks && l4_ipc_error(r, utcb))
+    printf("responder: ipc err (wait)\n");
+
+  l4_msgtag_t tag = l4_msgtag(0, 0, 0, 0);
+  while (1)
+    {
+      r = l4_ipc_reply_and_wait(utcb, tag, &label, L4_IPC_NEVER);
+      if (Enable_return_checks && l4_ipc_error(r, utcb))
+        printf("responder: ipc err (reply+wait)\n");
+    }
+
+  return NULL;
+}
+
+void syscall_bench(l4_cap_idx_t thread, unsigned cpu)
+{
+  l4_msgtag_t tag = l4_msgtag(L4_PROTO_THREAD, 1, 0, 0);
+  l4_utcb_t *utcb = l4_utcb();
+
+  PREPARE();
+
+  l4_utcb_mr_u(utcb)->mr[L4_THREAD_CONTROL_MR_IDX_FLAGS] = L4_THREAD_CONTROL_OP;
+
+  UNIT_TYPE(start);
+  UNIT_TYPE(end);
+  TAKE_TIME(start);
+  SYNC();
+  for (int i = 0; i < Num_rounds; ++i)
+    {
+      // This will go to the thread's control function and bail out
+      // early there.
+      l4_msgtag_t r = l4_ipc_call(thread, utcb, tag, L4_IPC_NEVER);
+      if (Enable_return_checks && l4_ipc_error(r, utcb))
+        printf("IPC error\n");
+    }
+  SYNC();
+  TAKE_TIME(end);
+
+  finished();
+
+  PRINT_RESULT(cpu, start, end, "syscall", 1);
+}

src/l4/pkg/ipcbench/src/ipc_common.h

@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re syscall benchmark program */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#pragma once
+
+#include <l4/sys/types.h>
+
+struct Caller_params
+{
+  l4_cap_idx_t responder_cap;
+  unsigned cpu;
+};
+
+void check_pthr_err(int r, char const *msg);
+
+void enumerate_cpus(void (*cb)(unsigned cpu, void *arg), void *arg);
+unsigned count_cpus(void);
+long run_thread(l4_cap_idx_t thread, unsigned cpu);
+
+void *fn_caller(void *cp);
+void *fn_responder(void *ignore);
+
+void syscall_bench(l4_cap_idx_t thread, unsigned cpu);
+
+void wait_for_start(void);
+void start(void);
+void set_completion_counter(unsigned val);
+void finished(void);

src/l4/pkg/ipcbench/src/ipcbench.c

@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re IPC benchmark program */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <pthread-l4.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "ipc_common.h"
+
+int main(int argc, char **argv)
+{
+  (void)argc;
+  (void)argv;
+
+  pthread_t thread_responder;
+  check_pthr_err(pthread_create(&thread_responder, NULL, fn_responder, NULL),
+                 "create responder thread");
+
+  // Wait for responder to be ready
+  l4_utcb_t *utcb = l4_utcb();
+  if (l4_ipc_error(l4_ipc_call(pthread_l4_cap(thread_responder), utcb,
+                               l4_msgtag(0, 0, 0, 0), L4_IPC_NEVER),
+                   utcb))
+    printf("Error syncing with responder thread!\n");
+
+  struct Caller_params cp = {pthread_l4_cap(thread_responder), 0};
+  start();
+  fn_caller(&cp);
+
+  check_pthr_err(pthread_cancel(thread_responder), "cancel responder thread");
+  check_pthr_err(pthread_join(thread_responder, NULL), "join responder thread");
+
+  return 0;
+}

src/l4/pkg/ipcbench/src/ipcbench_parallel.c

@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re IPC benchmark program */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <pthread-l4.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <l4/re/env.h>
+
+#include "ipc_common.h"
+
+struct Pair
+{
+  pthread_t caller_thread;
+  struct Caller_params caller_params;
+  pthread_t responder_thread;
+};
+
+struct Spawn_param
+{
+  unsigned idx;
+  unsigned num_cpus;
+  struct Pair *pairs;
+};
+
+static void spawn_pair(unsigned cpu, void *arg)
+{
+  struct Spawn_param *param = (struct Spawn_param *)arg;
+
+  if (param->idx >= param->num_cpus)
+    {
+      printf("CPU %u showed up late! Ignoring...\n", cpu);
+      return;
+    }
+
+  unsigned idx = param->idx++;
+  struct Pair *pair = &param->pairs[idx];
+
+  // Create threads without starting them. We want to put them directly on the
+  // right CPU...
+  pthread_attr_t attr;
+  check_pthr_err(pthread_attr_init(&attr), "pthread_attr_init");
+  attr.create_flags |= PTHREAD_L4_ATTR_NO_START;
+
+  // Create responder first
+  check_pthr_err(pthread_create(&pair->responder_thread, &attr, fn_responder,
+                                NULL),
+                 "create responder thread");
+
+  long err = run_thread(pthread_l4_cap(pair->responder_thread), cpu);
+  if (err)
+    {
+      printf("Error starting responder on CPU %u: %ld\n", cpu, err);
+      exit(1);
+    }
+
+  // Wait for responder to be ready
+  l4_utcb_t *utcb = l4_utcb();
+  if (l4_ipc_error(l4_ipc_call(pthread_l4_cap(pair->responder_thread), utcb,
+                               l4_msgtag(0, 0, 0, 0), L4_IPC_NEVER),
+                   utcb))
+    printf("Error syncing with responder thread!\n");
+
+  // Now we can create the caller
+  pair->caller_params.responder_cap = pthread_l4_cap(pair->responder_thread);
+  pair->caller_params.cpu = cpu;
+  check_pthr_err(pthread_create(&pair->caller_thread, &attr, fn_caller,
+                                &pair->caller_params),
+                 "create caller thread");
+
+  err = run_thread(pthread_l4_cap(pair->caller_thread), cpu);
+  if (err)
+    {
+      printf("Error starting caller on CPU %u: %ld\n", cpu, err);
+      exit(1);
+    }
+}
+
+int main(int argc, char **argv)
+{
+  (void)argc;
+  (void)argv;
+
+  unsigned num_cpus = count_cpus();
+  printf("Found %u CPUs. Measuring core-local IPC latency on all CPUs.\n",
+         num_cpus);
+
+  // Spawn IPC benchmark pairs on all CPUs.
+  struct Pair pairs[num_cpus];
+  struct Spawn_param sp = {0, num_cpus, pairs};
+  enumerate_cpus(&spawn_pair, &sp);
+
+  set_completion_counter(sp.idx);
+
+  start();
+
+  // Wait for tests to finish
+  for (unsigned i = 0; i < sp.idx; i++)
+    {
+      check_pthr_err(pthread_join(pairs[i].caller_thread, NULL),
+                     "join caller thread");
+      check_pthr_err(pthread_cancel(pairs[i].responder_thread),
+                     "cancel responder thread");
+      check_pthr_err(pthread_join(pairs[i].responder_thread, NULL),
+                     "join responder thread");
+    }
+
+  return 0;
+}

src/l4/pkg/ipcbench/src/ipcclient.c

@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re IPC benchmark program -- Client side */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <assert.h>
+#include <stdio.h>
+
+#include <l4/re/env.h>
+
+#include "measure.h"
+
+enum { Enable_return_checks = 1 };
+
+int main(int argc, char **argv)
+{
+  (void)argc;
+  (void)argv;
+
+  l4_cap_idx_t server = l4re_env_get_cap("comm");
+  assert(l4_is_valid_cap(server));
+
+  l4_utcb_t *utcb = l4_utcb();
+  l4_msgtag_t tag = l4_msgtag(0, 0, 0, 0);
+
+  l4_msgtag_t r = l4_ipc_call(server, utcb, tag, L4_IPC_NEVER);
+  if (l4_ipc_error(r, utcb))
+    printf("IPC error on initial rendevouz\n");
+
+  PREPARE();
+
+  UNIT_TYPE(start);
+  UNIT_TYPE(end);
+  TAKE_TIME(start);
+  SYNC();
+  for (int i = 0; i < Num_rounds; ++i)
+    {
+      r = l4_ipc_call(server, utcb, tag, L4_IPC_NEVER);
+      if (Enable_return_checks && l4_ipc_error(r, utcb))
+        printf("IPC error\n");
+    }
+  SYNC();
+  TAKE_TIME(end);
+
+  PRINT_RESULT(0, start, end, "IPC", 2);
+
+  return 0;
+}

src/l4/pkg/ipcbench/src/ipcserver.c

@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re IPC benchmark program -- Server side */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <assert.h>
+#include <stdio.h>
+
+#include <l4/sys/ipc.h>
+#include <l4/sys/rcv_endpoint.h>
+#include <l4/re/env.h>
+
+#include "measure.h"
+
+enum { Enable_return_checks = 1 };
+
+int main(int argc, char **argv)
+{
+  (void)argc;
+  (void)argv;
+
+  l4_cap_idx_t server = l4re_env_get_cap("comm");
+  assert(l4_is_valid_cap(server));
+
+  l4_msgtag_t r = l4_rcv_ep_bind_thread(server, l4re_env()->main_thread, 0x10);
+  assert(l4_error(r) == 0);
+
+  l4_utcb_t *utcb = l4_utcb();
+  l4_umword_t label;
+
+  r = l4_ipc_wait(utcb, &label, L4_IPC_NEVER);
+  if (Enable_return_checks && l4_ipc_error(r, utcb))
+    printf("IPC error (wait)\n");
+
+  l4_msgtag_t tag = l4_msgtag(0, 0, 0, 0);
+  while (1)
+    {
+      r = l4_ipc_reply_and_wait(utcb, tag, &label, L4_IPC_NEVER);
+      if (Enable_return_checks && l4_ipc_error(r, utcb))
+        printf("IPC error (reply+wait)\n");
+    }
+
+  return 0;
+}

src/l4/pkg/ipcbench/src/measure.h

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <l4/util/kip.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static inline void check_kernel_features(void)
+{
+  if (!l4_kip_kernel_has_feature(l4_kip(), "perf_cnt_user"))
+    {
+      printf("You need to enable CONFIG_PERF_CNT_USER in the microkernel.\n");
+      printf("This is required to use the performance measurement facilities\n"
+             "of the processor in this benchmark program.\n");
+      exit(1);
+    }
+}
+
+
+#if defined(__amd64__) || defined(__i686__)
+#include <l4/util/rdtsc.h>
+#include <l4/util/cpu.h>
+#if 0 /* rdpmc is not available in QEMU */
+#define NUMBERS 1
+#define UNIT_NAME(x) "cpu-cycles"
+#define TAKE_TIME(v) v[0] = l4_rdtsc()
+#define PREPARE() do {} while (0)
+#else
+#define NUMBERS 3
+#define UNIT_NAME(x) ({ char *__nn[NUMBERS] = {"insnret-units", "clk-units", "tsc-units" }; __nn[x]; })
+#define TAKE_TIME(v) do { v[0] = l4_rdpmc((1 << 30) | 0); v[1] = l4_rdpmc((1 << 30) | 1); v[2] = l4_rdpmc((1 << 30) | 2); } while (0)
+#define PREPARE() do { check_kernel_features(); } while (0)
+#endif
+#define UNIT_TYPE(t) l4_uint64_t t[NUMBERS]
+#define DIFF(idx, start, end) (end[idx] - start[idx])
+#define SYNC() l4util_cpu_capabilities()
+#elif defined(__aarch64__)
+
+#if 1 // < PMUv3, just cpu-cycles
+#define NUMBERS 1
+#define UNIT_NAME(x) "cpu-cycles"
+#define TAKE_TIME(v) asm volatile ("mrs %0, PMCCNTR_EL0" : "=r"(v))
+#define PREPARE() do { check_kernel_features(); asm volatile ("msr PMCCFILTR_EL0, %0" : : "r" (1UL << 27)); } while (0) // Enable counting in EL2 too
+#else
+#define NUMBERS 3
+#define UNIT_NAME(x) ({ char *__nn[NUMBERS] = {"CPU_CYCLES", "INST_RETIRED", "EXC_TAKEN" }; __nn[x]; })
+#define SET_EVENT(eventreg, eventtype) do { asm volatile ("msr " eventreg ", %0" : : "r" ((1UL << 27) | eventtype));  } while (0) // Enable counting in EL2 too
+#define READ_CNT(eventreg) ({ l4_uint64_t v; asm volatile("mrs %0, " eventreg : "=r"(v)); v; })
+#define PREPARE() do { check_kernel_features(); asm volatile("msr PMCNTENSET_EL0, %0" : : "r" (7 | (1UL << 31))); SET_EVENT("PMEVTYPER0_EL0", 0x11); SET_EVENT("PMEVTYPER1_EL0", 8); SET_EVENT("PMEVTYPER2_EL0", 9);  } while (0)
+#define TAKE_TIME(v) do { v[0] = READ_CNT("PMEVCNTR0_EL0"); v[1] = READ_CNT("PMEVCNTR1_EL0"); v[2] = READ_CNT("PMEVCNTR2_EL0"); } while (0)
+#endif
+
+#define UNIT_TYPE(t) l4_uint64_t t[NUMBERS]
+#define DIFF(idx, start, end) (end[idx] - start[idx])
+#define SYNC() do {} while (0)
+
+#elif defined(__arm__)
+
+#define NUMBERS 1
+#define UNIT_TYPE(x) l4_uint64_t x
+#define UNIT_NAME(x) "cpu-cycles"
+#define PREPARE() do { check_kernel_features(); asm volatile ("mcr p15, 0, %0, c14, c15, 7" : : "r" (1UL << 27)); } while (0) // Enable counting in EL2 too
+#define TAKE_TIME(v) asm volatile ("mrc p15, 0, %0, c9, c13, 0" : "=r" (v))
+#define DIFF(x, start, end) ((end) - (start))
+#define SYNC() do {} while (0)
+
+#else
+/* Generic */
+#include <time.h>
+
+#define NUMBERS 1
+#define UNIT_TYPE(x) l4_uint64_t x
+#define UNIT_NAME(x) "ns"
+#define PREPARE() do {} while (0)
+#define TAKE_TIME(v) ({ struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); v = ts.tv_sec * 1000000000 + ts.tv_nsec;})
+#define DIFF(x, start, end) ((end) - (start))
+#define SYNC() do {} while (0)
+#endif
+
+#define PRINT_RESULT(cpu, start, end, opname, factor) \
+  for (unsigned n = 0; n < NUMBERS; ++n) \
+    printf("CPU %2u done %d " opname "s in %lld %s, %lld %s/" opname "\n", \
+           cpu, Num_rounds * factor, DIFF(n, start, end), UNIT_NAME(n), \
+           DIFF(n, start, end) / (Num_rounds * factor), UNIT_NAME(n)); \
+
+enum { Num_rounds = 300000 };

src/l4/pkg/ipcbench/src/syscallbench.c

@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re syscall benchmark program */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <l4/re/env.h>
+
+#include "ipc_common.h"
+
+int main(int argc, char **argv)
+{
+  (void)argc;
+  (void)argv;
+
+  syscall_bench(l4re_env()->main_thread, 0);
+
+  return 0;
+}

src/l4/pkg/ipcbench/src/syscallbench_parallel.c

@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: MIT */
+/* Small and simple L4Re syscall benchmark program */
+/* by Adam Lackorzynski <adam@l4re.org> */
+
+#include <stdio.h>
+
+#include <pthread-l4.h>
+
+#include "ipc_common.h"
+
+
+struct Thread
+{
+  pthread_t thread;
+  unsigned cpu;
+};
+
+struct Spawn_param
+{
+  unsigned idx;
+  unsigned num_cpus;
+  struct Thread *threads;
+};
+
+static void *fn_syscall(void *arg)
+{
+  struct Thread *thread = (struct Thread *)arg;
+  wait_for_start();
+  syscall_bench(pthread_l4_cap(thread->thread), thread->cpu);
+  return NULL;
+}
+
+static void spawn_threads(unsigned cpu, void *sp)
+{
+  struct Spawn_param *param = (struct Spawn_param *)sp;
+  if (param->idx >= param->num_cpus)
+    {
+      printf("CPU %u showed up late! Ignoring...\n", cpu);
+      return;
+    }
+
+  unsigned idx = param->idx++;
+  struct Thread *thread = &param->threads[idx];
+
+  thread->cpu = cpu;
+
+  // Create thread without starting. We want to put it directly on the right
+  // CPU...
+  pthread_attr_t attr;
+  check_pthr_err(pthread_attr_init(&attr), "pthread_attr_init");
+  attr.create_flags |= PTHREAD_L4_ATTR_NO_START;
+  check_pthr_err(pthread_create(&thread->thread, &attr, fn_syscall,
+                                thread),
+                 "create thread");
+
+  long err = run_thread(pthread_l4_cap(thread->thread), cpu);
+  if (err)
+    {
+      printf("Error starting responder on CPU %u: %ld\n", cpu, err);
+      exit(1);
+    }
+}
+
+int main(int argc, char **argv)
+{
+  (void)argc;
+  (void)argv;
+
+  unsigned num_cpus = count_cpus();
+  printf("Found %u CPUs. Measuring syscall latency on all CPUs.\n",
+         num_cpus);
+
+  // Spawn IPC benchmark pairs on all CPUs.
+  struct Thread threads[num_cpus];
+  struct Spawn_param sp = {0, num_cpus, threads};
+  enumerate_cpus(&spawn_threads, &sp);
+
+  set_completion_counter(sp.idx);
+
+  start();
+
+  // Wait for tests to finish
+  for (unsigned i = 0; i < sp.idx; i++)
+    check_pthr_err(pthread_join(threads[i].thread, NULL), "join thread");
+
+  return 0;
+}