LiveUpdate support for individually resumable partitions

Author: fwsGonzo

lib/LiveUpdate/CMakeLists.txt

@@ -19,8 +19,8 @@ add_custom_target(hotswap64 DEPENDS hotswap64.bin)
 
 # LiveUpdate static library
 add_library(liveupdate STATIC
-    storage.cpp update.cpp resume.cpp rollback.cpp hotswap.cpp
-    serialize_tcp.cpp hotswap64_blob.asm
+    storage.cpp partition.cpp update.cpp resume.cpp rollback.cpp
+    serialize_tcp.cpp hotswap.cpp hotswap64_blob.asm
   )
 add_dependencies(liveupdate hotswap64)
 install(TARGETS liveupdate DESTINATION includeos/${ARCH}/lib)

lib/LiveUpdate/liveupdate.hpp

@@ -77,10 +77,10 @@ struct LiveUpdate
   // Attempt to restore existing stored entries.
   // Returns false if there was nothing there. or if the process failed
   // to be sure that only failure can return false, use is_resumable first
-  static bool resume(std::string key, resume_func default_handler);
+  static void resume(std::string key, resume_func default_handler);
 
   // When explicitly resuming from heap, heap overrun checks are disabled
-  static bool resume_from_heap(void* location, resume_func default_handler);
+  static void resume_from_heap(void* location, std::string key, resume_func);
 
   // Retrieve the recorded length, in bytes, of a valid storage area
   // Throws std::runtime_error when something bad happens

lib/LiveUpdate/partition.cpp

@@ -0,0 +1,74 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 IncludeOS AS, Oslo, Norway
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+/**
+ * Master thesis
+ * by Alf-Andre Walla 2016-2017
+ *
+**/
+#include "storage.hpp"
+#include <util/crc32.hpp>
+
+inline uint32_t liu_crc32(const void* buf, size_t len)
+{
+  return crc32_fast(buf, len);
+}
+
+uint32_t partition_header::generate_checksum(const char* vla) const {
+  return liu_crc32(&vla[this->offset], this->length);
+}
+
+int storage_header::create_partition(const std::string key)
+{
+  if (key.size() > sizeof(partition_header::name)-1)
+      throw std::length_error("Key '" + key + "' too long");
+  if (partitions >= ptable.size())
+      throw std::out_of_range("Partition table is full");
+  auto& part = ptable.at(partitions);
+  snprintf(part.name, sizeof(part.name), "%s", key.c_str());
+  part.offset = this->length;
+  return partitions++;
+}
+int storage_header::find_partition(const char* key)
+{
+  for (uint32_t p = 0; p < this->partitions; p++)
+  {
+    auto& part = ptable.at(p);
+    if (strncmp(part.name, key, sizeof(part.name)))
+    {
+      uint32_t chsum = part.generate_checksum(this->vla);
+      if (part.crc == chsum) {
+        return p;
+      }
+      throw std::runtime_error("Invalid CRC in partition '" + std::string(key) + "'");
+    }
+  }
+  throw std::out_of_range("Could not find partition named " + std::string(key));
+}
+void storage_header::finish_partition(int p)
+{
+  // make sure partition ends properly
+  this->add_end();
+  // write length and crc
+  auto& part = ptable.at(p);
+  part.length = this->length - part.offset;
+  part.crc = part.generate_checksum(this->vla);
+}
+void storage_header::zero_partition(int p)
+{
+  auto& part = ptable.at(p);
+  memset(&vla[part.offset], 0, part.length);
+  memset(&part, 0, sizeof(partition_header));
+}

lib/LiveUpdate/resume.cpp

@@ -34,6 +34,7 @@ extern char* heap_end;
 
 namespace liu
 {
+static void resume_begin(storage_header&, const char*, LiveUpdate::resume_func);
 static std::map<uint16_t, LiveUpdate::resume_func> resume_funcs;
 
 bool LiveUpdate::is_resumable()
@@ -45,44 +46,40 @@ bool LiveUpdate::is_resumable(void* location)
   return ((storage_header*) location)->validate();
 }
 
-static bool resume_helper(void* location, LiveUpdate::resume_func func)
+static void resume_helper(void* location, std::string key, LiveUpdate::resume_func func)
 {
   // check if an update has occurred
-  if (!LiveUpdate::is_resumable(location)) return false;
+  if (!LiveUpdate::is_resumable(location))
+      throw std::runtime_error("Trying to resume from invalid storage area");
 
   LPRINT("* Restoring data...\n");
   // restore connections etc.
-  extern bool resume_begin(storage_header&, LiveUpdate::resume_func);
-  return resume_begin(*(storage_header*) location, func);
+  resume_begin(*(storage_header*) location, key.c_str(), func);
 }
-bool LiveUpdate::resume(std::string key, resume_func func)
+void LiveUpdate::resume(std::string key, resume_func func)
 {
   void* location = OS::liveupdate_storage_area();
   /// memory sanity check
   if (heap_end >= (char*) location) {
     fprintf(stderr,
         "WARNING: LiveUpdate storage area inside heap (margin: %ld)\n",
 		     (long int) (heap_end - (char*) location));
-    return false;
+    throw std::runtime_error("LiveUpdate storage area inside heap");
   }
-  return resume_helper(location, func);
+  resume_helper(location, key, func);
 }
-bool LiveUpdate::resume_from_heap(void* location, LiveUpdate::resume_func func)
+void LiveUpdate::resume_from_heap(void* location, std::string key, LiveUpdate::resume_func func)
 {
-  return resume_helper(location, func);
+  resume_helper(location, key, func);
 }
 
-bool resume_begin(storage_header& storage, LiveUpdate::resume_func func)
+void resume_begin(storage_header& storage, const char* key, LiveUpdate::resume_func func)
 {
-  /// restore each entry one by one, calling registered handlers
-  auto num_ents = storage.get_entries();
-  if (num_ents > 1) {
-    LPRINT("* Resuming %d stored entries\n", num_ents-1);
-  } else {
-    LPRINT("* No stored entries to resume\n");
-  }
+  int p = storage.find_partition(key);
+  LPRINT("* Resuming from partition %d\n", p);
 
-  for (auto* ptr = storage.begin(); ptr->type != TYPE_END;)
+  /// restore each entry one by one, calling registered handlers
+  for (auto* ptr = storage.begin(p); ptr->type != TYPE_END;)
   {
     auto* oldptr = ptr;
     // resume wrapper
@@ -102,10 +99,8 @@ bool resume_begin(storage_header& storage, LiveUpdate::resume_func func)
   }
   /// wake all the slumbering IP stacks
   serialized_tcp::wakeup_ip_networks();
-  /// zero out all the state for security reasons
-  storage.zero();
-
-  return true;
+  /// zero out the partition for security reasons
+  storage.zero_partition(p);
 }
 
 void LiveUpdate::on_resume(uint16_t id, resume_func func)

lib/LiveUpdate/storage.cpp

@@ -25,6 +25,11 @@
 #include <cassert>
 //#define VERIFY_MEMORY
 
+inline uint32_t liu_crc32(const void* buf, size_t len)
+{
+  return crc32_fast(buf, len);
+}
+
 const uint64_t storage_header::LIVEUPD_MAGIC = 0xbaadb33fdeadc0de;
 
 storage_header::storage_header()
@@ -33,11 +38,6 @@ storage_header::storage_header()
   //printf("%p --> %#llx\n", this, value);
 }
 
-inline uint32_t liu_crc32(const void* buf, size_t len)
-{
-  return crc32_fast(buf, len);
-}
-
 void storage_header::add_marker(uint16_t id)
 {
   create_entry(TYPE_MARKER, id, 0);
@@ -134,7 +134,9 @@ void storage_header::finalize()
 {
   if (this->magic != LIVEUPD_MAGIC)
       throw std::runtime_error("Magic field invalidated during store process");
-  add_end();
+  // add end if missing
+  if (this->length == 0) this->add_end();
+  // generate checksum for header
   this->crc = generate_checksum();
 }
 bool storage_header::validate() noexcept
@@ -153,7 +155,7 @@ uint32_t storage_header::generate_checksum() noexcept
   this->crc         = 0;
 
   const char* begin = (const char*) this;
-  size_t      len   = sizeof(storage_header) + this->length;
+  size_t      len   = sizeof(storage_header);
   uint32_t checksum = liu_crc32(begin, len);
 
   this->crc = crc_copy;
@@ -166,9 +168,9 @@ void storage_header::zero()
   assert(this->magic == 0);
 }
 
-storage_entry* storage_header::begin()
+storage_entry* storage_header::begin(int p)
 {
-  return (storage_entry*) vla;
+  return (storage_entry*) &vla[ptable.at(p).offset];
 }
 storage_entry* storage_header::next(storage_entry* ptr)
 {

lib/LiveUpdate/storage.hpp

@@ -20,6 +20,7 @@
 **/
 #pragma once
 #include <cstdint>
+#include <array>
 #include <string>
 #include <vector>
 #include <delegate>
@@ -86,6 +87,16 @@ struct storage_entry
   uint32_t       checksum() const;
 };
 
+struct partition_header
+{
+  uint32_t generate_checksum(const char* vla) const;
+
+  char      name[16] = {0};
+  uint32_t  crc = 0;
+  uint32_t  offset = 0;
+  uint32_t  length = 0;
+};
+
 struct storage_header
 {
   typedef delegate<int(char*)> construct_func;
@@ -102,6 +113,10 @@ struct storage_header
   }
 
   storage_header();
+  int  create_partition(std::string key);
+  int  find_partition(const char*);
+  void finish_partition(int);
+  void zero_partition(int);
 
   void add_marker(uint16_t id);
   void add_int   (uint16_t id, int value);
@@ -113,7 +128,7 @@ struct storage_header
   void add_string_vector(uint16_t id, const std::vector<std::string>& vec);
   void add_end();
 
-  storage_entry* begin();
+  storage_entry* begin(int p);
   storage_entry* next(storage_entry*);
 
   template <typename... Args>
@@ -138,6 +153,8 @@ struct storage_header
   uint32_t crc;
   uint32_t entries = 0;
   uint32_t length  = 0;
+  std::array<partition_header, 16> ptable;
+  uint32_t partitions = 0;
   char     vla[0];
 };
 

lib/LiveUpdate/update.cpp

@@ -276,8 +276,13 @@ size_t update_store_data(void* location, const buffer_t* blob)
   /// callback for storing stuff, if provided
   for (const auto& pair : storage_callbacks)
   {
+    // create partition
+    int p = storage->create_partition(pair.first);
+    // run serialization process
     Storage wrapper {*storage};
     pair.second(wrapper, blob);
+    // add end for partition
+    storage->finish_partition(p);
   }
 
   /// finalize

lib/uplink/ws_uplink.cpp

@@ -74,8 +74,7 @@ namespace uplink {
     if(liu::LiveUpdate::is_resumable())
     {
       MYINFO("Found resumable state, try restoring...");
-      auto success = liu::LiveUpdate::resume("uplink", {this, &WS_uplink::restore});
-      CHECK(success, "Success");
+      liu::LiveUpdate::resume("uplink", {this, &WS_uplink::restore});
     }
 
     client_ = std::make_unique<http::Client>(inet.tcp(),