merged

Author: alfreb

.gitignore

@@ -14,6 +14,7 @@
 *.d
 *.pyc
 *.ropeproject
+*.xml
 nbproject
 
 !Dockerfile

.travis.yml

@@ -0,0 +1,21 @@
+matrix:
+  include:
+  - os: osx
+    env: COMPILER='clang++'
+    osx_image: xcode8.2
+  - os: osx
+    env: COMPILER='clang++'
+    osx_image: xcode8
+
+install:
+  - if [ $TRAVIS_OS_NAME == osx ]; then brew update ; brew uninstall --force cmake; brew install cmake ; fi
+  - git submodule update --init --recursive
+
+before_script:
+  - export CXX=$COMPILER
+  - cd test
+  - mkdir darwin && cd darwin
+
+script:
+  - cmake .. -DTRAVIS=ON -DEXTRA_TESTS=ON
+  - make VERBOSE=1 && ./unittests --pass

CMakeLists.txt

@@ -14,7 +14,7 @@ set(SCRIPTS ${CMAKE_INSTALL_PREFIX}/includeos/scripts)
 if(CMAKE_COMPILER_IS_GNUCC)
 	# currently gcc is not supported due to problems cross-compiling a unikernel
 	# (i.e., building a 32bit unikernel (only supported for now) on a 64bit system)
-	message(FATAL_ERROR "Building IncludeOS with gcc is not currently supported. Please clean-up build directory and configure for clang through CC and CXX environmental variables.")
+#	message(FATAL_ERROR "Building IncludeOS with gcc is not currently supported. Please clean-up build directory and configure for clang through CC and CXX environmental variables.")
 endif(CMAKE_COMPILER_IS_GNUCC)
 
 # create OS version string from git describe (used in CXX flags)
@@ -23,10 +23,21 @@ execute_process(COMMAND git describe --dirty
 	        OUTPUT_VARIABLE OS_VERSION)
 string(STRIP ${OS_VERSION} OS_VERSION)
 
+option(cpu_feat_vanilla "Restrict use of CPU features to vanilla" ON)
+if(cpu_feat_vanilla)
+  include("cmake/vanilla.cmake")
+  set(DEFAULT_SETTINGS_CMAKE "vanilla.cmake") # for service cmake
+  set(DEFAULT_VM "vm.vanilla.json") # vmrunner
+else()
+  include("cmake/cpu_feat.cmake")
+  set(DEFAULT_SETTINGS_CMAKE "cpu_feat.cmake") # for service cmake
+  set(DEFAULT_VM "vm.cpu_feat.json") # vmrunner
+endif(cpu_feat_vanilla)
+
 # create random hex string as stack protector canary
 string(RANDOM LENGTH 8 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE)
 
-set(CAPABS "-msse3 -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE} ")
+set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}")
 
 # Various global defines
 # * NO_DEBUG disables output from the debug macro
@@ -57,7 +68,7 @@ init_submodule(mod/uzlib)
 set(OPTIMIZE "-O2")
 
 if(debug OR debug-info OR debug-all)
-	set(CAPABS "${CAPABS} -O0")
+	set(OTPIMIZE "-O0")
 endif(debug OR debug-info OR debug-all)
 
 if(minimal)
@@ -81,24 +92,41 @@ endif(silent)
 # Append optimization level
 set(CAPABS "${CAPABS} ${OPTIMIZE}")
 
-# these kinda work with llvm
-set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
-set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+if (CMAKE_COMPILER_IS_GNUCC)
+  # gcc/g++ settings
+  set(CMAKE_CXX_FLAGS "-m32 -MMD ${CAPABS} ${WARNS} -Wno-frame-address -nostdlib -c -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
+  set(CMAKE_C_FLAGS "-m32 -MMD ${CAPABS} ${WARNS} -nostdlib -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+else()
+  # these kinda work with llvm
+  set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
+  set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+endif()
 
 # either download or cross-compile needed libraries
-option(from_bundle "Download and use pre-compiled libraries for cross-comilation" ON)
+#option(from_bundle "Download and use pre-compiled libraries for cross-comilation" ON)
+set(BUNDLE_LOC "" CACHE STRING "Local path of bundle with pre-compile libraries")
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/cross_compiled_libraries.txt)
 
-add_subdirectory(mod)
+# Botan Crypto & TLS
+# Note: Include order matters!
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/botan.cmake)
 
+#
+# Subprojects
+#
+add_subdirectory(mod)
 add_subdirectory(src)
 
+#
+# External projects
+#
 option(vmbuild "Build and install vmbuild and elf_syms" ON)
 if(vmbuild)
   # Install vmbuilder as an external project
   ExternalProject_Add(vmbuild
     PREFIX vmbuild # Build where
     SOURCE_DIR ${INCLUDEOS_ROOT}/vmbuild # Where is project located
+    BINARY_DIR ${INCLUDEOS_ROOT}/vmbuild/build
     INSTALL_DIR ${BIN} # Where to install
     CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> # Pass installation folder
     )
@@ -128,6 +156,7 @@ if(tests)
   ExternalProject_Add(unittests
     PREFIX unittests
     SOURCE_DIR ${INCLUDEOS_ROOT}/test
+    BINARY_DIR unittests
     CMAKE_ARGS -DINCLUDEOS_ROOT=${INCLUDEOS_ROOT} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
     )
   #add_subdirectory(test)
@@ -145,10 +174,24 @@ if(rapidjson)
   install(DIRECTORY mod/rapidjson/include/rapidjson DESTINATION includeos/include)
 endif(rapidjson)
 
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/etc/service.cmake DESTINATION includeos)
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/etc/library.cmake DESTINATION includeos)
-install(DIRECTORY vmrunner DESTINATION includeos/)
+
+#
+# Installation
+#
+
+# Install cmake files
+install(FILES etc/service.cmake DESTINATION includeos)
+install(FILES etc/library.cmake DESTINATION includeos)
+install(FILES cmake/${DEFAULT_SETTINGS_CMAKE} DESTINATION includeos RENAME settings.cmake) # cpu_feat_vanilla opt
+
+# Install vmrunner
+install(DIRECTORY vmrunner DESTINATION includeos)
+install(FILES vmrunner/${DEFAULT_VM} DESTINATION includeos/vmrunner/ RENAME vm.default.json) # cpu_feat_vanilla opt
+
+# Install toolchain
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/i686-elf-toolchain.cmake DESTINATION includeos)
+
+# Install seed
 install(DIRECTORY seed/ DESTINATION includeos/seed)
 
 # Install boot util

README.md

@@ -22,15 +22,15 @@ IncludeOS is free software, with "no warranties or restrictions of any kind".
 
 ## Build status
 
-|        | Build from bundle                                                                                                                                             |
-|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Master | [![Build Status](https://jenkins.includeos.org/buildStatus/icon?job=shield_master_bundle)](https://jenkins.includeos.org/job/shield_master_bundle/) |
-| Dev    | [![Build Status](https://jenkins.includeos.org/buildStatus/icon?job=shield_dev_bundle)](https://jenkins.includeos.org/job/shield_dev_bundle/)      |
+|        | Build from bundle | Integration tests |
+|--------|-------------------|-------------------|
+| Master | [![Build Status](https://jenkins.includeos.org/buildStatus/icon?job=shield_master_bundle)](https://jenkins.includeos.org/job/shield_master_bundle/) |  Coming soon |
+| Dev    | [![Build Status](https://jenkins.includeos.org/buildStatus/icon?job=shield_dev_bundle)](https://jenkins.includeos.org/job/shield_dev_bundle/)      | [![Jenkins tests](https://img.shields.io/jenkins/t/https/jenkins.includeos.org/shield_dev_integration_tests.svg)](https://jenkins.includeos.org/job/shield_dev_integration_tests/) |
 
 ### Key features
 
 * **Extreme memory footprint**: A minimal bootable image, including bootloader, operating system components and a complete C++ standard library is currently 707K when optimized for size.
-* **KVM and VirtualBox support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on any modern x86 CPUs). In principle IncludeOS should run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), which power the [OpenStack IaaS cloud](https://www.openstack.org/), and [VirtualBox](https://www.virtualbox.org), which means that you can run your IncludeOS service on both Linux, Microsoft Windows and macOS.
+* **KVM, VirtualBox and VMWare support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on any modern x86 CPUs). In principle IncludeOS should run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), which power the [OpenStack IaaS cloud](https://www.openstack.org/), and [VirtualBox](https://www.virtualbox.org), which means that you can run your IncludeOS service on both Linux, Microsoft Windows and macOS.
 * **C++11/14 support**
     * Full C++11/14 language support with [clang](http://clang.llvm.org) v3.8 and later.
     * Standard C++ library (STL) [libc++](http://libcxx.llvm.org) from [LLVM](http://llvm.org/).

api/arch

@@ -0,0 +1,45 @@
+// -*-C++-*-
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2015 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#ifndef INCLUDEOS_ARCH_HEADER
+#define INCLUDEOS_ARCH_HEADER
+
+#include <cstddef>
+#include <cstdint>
+
+extern void __arch_init();
+extern void __arch_poweroff();
+extern void __arch_reboot();
+extern void __arch_enable_legacy_irq(uint8_t);
+extern void __arch_disable_legacy_irq(uint8_t);
+
+#ifdef ARCH_X86
+inline uint64_t __arch_cpu_cycles() noexcept {
+  uint64_t ret;
+  asm("rdtsc" : "=A" (ret));
+  return ret;
+}
+#else
+inline uint64_t __arch_cpu_cycles() noexcept {
+  return 0;
+}
+#endif
+
+#endif
+

api/common

@@ -1,4 +1,4 @@
-// -*-C++-*-
+// -*- C++ -*-
 // This file is a part of the IncludeOS unikernel - www.includeos.org
 //
 // Copyright 2015 Oslo and Akershus University College of Applied Sciences
@@ -53,27 +53,11 @@
 #undef Expects
 #undef Ensures
 
-#define Expects(cond)                                       \
-  do {                                                      \
-    if (UNLIKELY(not (cond))) {                             \
-      std::cerr << "OS: Precondition (" OS_STRINGIFY(cond)  \
-                << ") failed...\n"  << "\t...in function: " \
-                << __func__ << " @" << __FILE__ << ":"      \
-                << __LINE__ << '\n';                        \
-      std::terminate();                                     \
-    }                                                       \
-  } while(0)
+#define Expects(cond) \
+  assert(LIKELY(cond) && "OS: Precondition failed")
 
-#define Ensures(cond)                                       \
-  do {                                                      \
-    if (UNLIKELY(not (cond))) {                             \
-      std::cerr << "OS: Postcondition (" OS_STRINGIFY(cond) \
-                << ") failed...\n"  << "\t...in function: " \
-                << __func__ << " @" << __FILE__ << ":"      \
-                << __LINE__ << '\n';                        \
-      std::terminate();                                     \
-    }                                                       \
-  } while(0)
+#define Ensures(cond) \
+  assert(LIKELY(cond) && "OS: Postcondition failed")
 
 #endif //< defined(OS_TERMINATE_ON_CONTRACT_VIOLATION)
 

api/fs/common.hpp

@@ -28,9 +28,8 @@
 
 namespace fs {
 
-  // Generic structure for directory entries
   struct Dirent;
-
+  struct File_system;
 
   /**
    * @brief Type used as a building block to represent buffers
@@ -185,24 +184,23 @@ namespace fs {
   /** @var no_error: Always returns boolean false when used in expressions */
   extern error_t no_error;
 
-    /** Async function types **/
-  using on_init_func  = delegate<void(error_t)>;
+  /** Async function types **/
+  using on_init_func  = delegate<void(error_t, File_system&)>;
   using on_ls_func    = delegate<void(error_t, dirvec_t)>;
   using on_read_func  = delegate<void(error_t, buffer_t, uint64_t)>;
-  using on_stat_func  = delegate<void(error_t, const Dirent&)>;
+  using on_stat_func  = delegate<void(error_t, Dirent)>;
 
 
-  struct List {
+  struct List
+  {
     error_t  error;
     dirvec_t entries;
     auto begin() { return entries->begin(); }
     auto end() { return entries->end(); }
     auto cbegin() { return entries->cbegin(); }
     auto cend() { return entries->cend(); }
-
   };
 
+} //< fs
 
-} //< namespace fs
-
-#endif //< FS_ERROR_HPP
+#endif //< FS_COMMON_HPP

api/fs/disk.hpp

@@ -42,7 +42,7 @@ namespace fs {
 
     struct Partition;
     using on_parts_func = delegate<void(fs::error_t, std::vector<Partition>&)>;
-    using on_init_func = delegate<void(fs::error_t)>;
+    using on_init_func  = delegate<void(fs::error_t, File_system&)>;
     using lba_t = uint32_t;
 
     enum partition_t {

api/fs/filesystem.hpp

@@ -27,10 +27,6 @@ namespace fs {
   struct Dirent;
 
   struct File_system {
-
-    /** Initialize this filesystem with LBA at @base_sector */
-    virtual void init(uint64_t lba, uint64_t size, on_init_func on_init) = 0;
-
     /** Get unique (per device type) device id for underlying device.*/
     virtual Device_id device_id() = 0;
 
@@ -76,6 +72,9 @@ namespace fs {
     /** Returns the name of this filesystem */
     virtual std::string name() const = 0;
 
+    /** Initialize this filesystem with LBA at @base_sector */
+    virtual void init(uint64_t lba, uint64_t size, on_init_func on_init) = 0;
+
     /** Default destructor */
     virtual ~File_system() noexcept = default;
   }; //< class File_system

api/fs/vfs.hpp

@@ -18,14 +18,15 @@
 #ifndef FS_VFS_HPP
 #define FS_VFS_HPP
 
-#include <memory>
 #include <fs/disk.hpp>
 #include <fs/filesystem.hpp>
 #include <fs/path.hpp>
 #include <hw/devices.hpp>
-#include <typeinfo>
-#include <stdexcept>
 #include <algorithm>
+#include <map>
+#include <memory>
+#include <stdexcept>
+#include <typeinfo>
 
 // Demangle
 extern "C" char* __cxa_demangle(const char* mangled_name,