smp: Restructure task arrays

Author: fwsGonzo

api/smp

@@ -114,8 +114,8 @@ public:
   // execute @func on another CPU core
   // call @done back on main CPU when task returns
   // use signal() to broadcast work should begin
-  static void add_task(task_func func, done_func done, int cpu = -1);
-  static void add_task(task_func func, int cpu = -1);
+  static void add_task(task_func func, done_func done, int cpu = 0);
+  static void add_task(task_func func, int cpu = 0);
   // execute a function on the main cpu
   static void add_bsp_task(done_func func);
 

src/platform/x86_pc/apic_revenant.cpp

@@ -12,10 +12,10 @@ extern "C" void   lapic_exception_handler();
 #define INFO(FROM, TEXT, ...) printf("%13s ] " TEXT "\n", "[ " FROM, ##__VA_ARGS__)
 
 using namespace x86;
-smp_stuff smp;
-SMP_ARRAY<smp_stuff> smp_local;
+smp_stuff smp_main;
+SMP_ARRAY<smp_system_stuff> smp_system;
 
-static bool revenant_task_doer(smp_stuff& system)
+static bool revenant_task_doer(smp_system_stuff& system)
 {
   // grab hold on task list
   lock(system.tlock);
@@ -39,26 +39,29 @@ static bool revenant_task_doer(smp_stuff& system)
   if (task.done)
   {
     // NOTE: specifically pushing to 'smp' here, and not 'system'
-    lock(smp.flock);
-    smp.completed.push_back(std::move(task.done));
-    unlock(smp.flock);
+    lock(smp_main.flock);
+    smp_main.completed.push_back(std::move(task.done));
+    unlock(smp_main.flock);
     return true;
   }
   // we did work, but we aren't going to signal back
   return false;
 }
 static void revenant_task_handler()
 {
-  bool work_done = false;
-  while (true) {
-    // global tasks
-    bool did_something = revenant_task_doer(smp);
-    work_done = work_done || did_something;
+  bool work_done     = false;
+  bool did_something = false;
+  do {
     // cpu-specific tasks
-    did_something = revenant_task_doer(PER_CPU(smp_local));
+    did_something = revenant_task_doer(PER_CPU(smp_system));
     work_done = work_done || did_something;
-    if (did_something == false) break;
-  }
+    // global tasks (by taking from index 0)
+    did_something = revenant_task_doer(smp_system[0]);
+    work_done = work_done || did_something;
+    // continue as long as something was done
+    // because there could be more
+  } while (did_something);
+  // if we did any work with done functions, signal back
   if (work_done) {
     x86::APIC::get().send_bsp_intr();
   }
@@ -91,7 +94,7 @@ void revenant_main(int cpu)
   ::SMP::init_task();
 
   // signal that the revenant has started
-  smp.boot_barrier.inc();
+  smp_main.boot_barrier.inc();
 
   while (true)
   {

src/platform/x86_pc/apic_revenant.hpp

@@ -25,26 +25,29 @@
 
 extern "C" void revenant_main(int);
 
-struct smp_stuff
-{
-  struct task {
-    task(SMP::task_func a,
-         SMP::done_func b)
-      : func(a), done(b) {}
+struct smp_task {
+  smp_task(SMP::task_func a,
+           SMP::done_func b)
+   : func(a), done(b) {}
 
-    SMP::task_func func;
-    SMP::done_func done;
-  };
+  SMP::task_func func;
+  SMP::done_func done;
+};
 
+struct smp_stuff
+{
   minimal_barrier_t boot_barrier;
 
-  spinlock_t tlock;
-  std::deque<task> tasks;
-
   spinlock_t flock;
   std::deque<SMP::done_func> completed;
 };
-extern smp_stuff smp;
-extern SMP_ARRAY<smp_stuff> smp_local;
+extern smp_stuff smp_main;
+
+struct smp_system_stuff
+{
+  spinlock_t tlock;
+  std::deque<smp_task> tasks;
+};
+extern SMP_ARRAY<smp_system_stuff> smp_system;
 
 #endif

src/platform/x86_pc/smp.cpp

@@ -75,7 +75,7 @@ void SMP::init()
       boot->stack_base, boot->stack_size, sizeof(boot->worker_addr));
 
   // reset barrier
-  smp.boot_barrier.reset(1);
+  smp_main.boot_barrier.reset(1);
 
   auto& apic = x86::APIC::get();
   // turn on CPUs
@@ -98,7 +98,7 @@ void SMP::init()
   }
 
   // wait for all APs to start
-  smp.boot_barrier.spin_wait(CPUcount);
+  smp_main.boot_barrier.spin_wait(CPUcount);
   INFO("SMP", "All %u APs are online now\n", CPUcount);
 
   // subscribe to IPIs
@@ -116,10 +116,10 @@ void SMP::init()
 std::vector<smp_done_func> SMP::get_completed()
 {
   std::vector<smp_done_func> done;
-  lock(smp.flock);
-  for (auto& func : smp.completed) done.push_back(func);
-  smp.completed.clear(); // MUI IMPORTANTE
-  unlock(smp.flock);
+  lock(smp_main.flock);
+  for (auto& func : smp_main.completed) done.push_back(func);
+  smp_main.completed.clear(); // MUI IMPORTANTE
+  unlock(smp_main.flock);
   return done;
 }
 
@@ -154,49 +154,33 @@ void ::SMP::init_task()
 void ::SMP::add_task(smp_task_func task, smp_done_func done, int cpu)
 {
 #ifdef INCLUDEOS_SINGLE_THREADED
-  assert(cpu == -1 || cpu == 0);
+  assert(cpu == 0);
   task(); done();
 #else
-  if (cpu == -1)
-  {
-    lock(smp.tlock);
-    smp.tasks.emplace_back(std::move(task), std::move(done));
-    unlock(smp.tlock);
-  }
-  else {
-    lock(smp_local[cpu].tlock);
-    smp_local[cpu].tasks.emplace_back(std::move(task), std::move(done));
-    unlock(smp_local[cpu].tlock);
-  }
+  lock(smp_system[cpu].tlock);
+  smp_system[cpu].tasks.emplace_back(std::move(task), std::move(done));
+  unlock(smp_system[cpu].tlock);
 #endif
 }
 void ::SMP::add_task(smp_task_func task, int cpu)
 {
 #ifdef INCLUDEOS_SINGLE_THREADED
-  assert(cpu == -1 || cpu == 0);
+  assert(cpu == 0);
   task();
 #else
-  if (cpu == -1)
-  {
-    lock(smp.tlock);
-    smp.tasks.emplace_back(std::move(task), nullptr);
-    unlock(smp.tlock);
-  }
-  else {
-    lock(smp_local[cpu].tlock);
-    smp_local[cpu].tasks.emplace_back(std::move(task), nullptr);
-    unlock(smp_local[cpu].tlock);
-  }
+  lock(smp_system[cpu].tlock);
+  smp_system[cpu].tasks.emplace_back(std::move(task), nullptr);
+  unlock(smp_system[cpu].tlock);
 #endif
 }
 void ::SMP::add_bsp_task(smp_done_func task)
 {
 #ifdef INCLUDEOS_SINGLE_THREADED
   task();
 #else
-  lock(smp.flock);
-  smp.completed.push_back(std::move(task));
-  unlock(smp.flock);
+  lock(smp_main.flock);
+  smp_main.completed.push_back(std::move(task));
+  unlock(smp_main.flock);
   x86::APIC::get().send_bsp_intr();
 #endif
 }