Merge pull request #1362 from hioa-cs/dev

Merge dev

Author: alfreb

.editorconfig

@@ -21,3 +21,6 @@ indent_size = 4
 
 [boot]
 indent_size = 4
+
+[*.asm]
+indent_size = 4

.gitignore

@@ -14,6 +14,7 @@
 *.d
 *.pyc
 *.ropeproject
+*.xml
 nbproject
 
 !Dockerfile
@@ -38,7 +39,9 @@ src/compile_commands.json
 /examples/demo_service/IncludeOS_Demo_Service
 /seed/My_Service
 
-build
+build/
+build_i686/
+build_x86_64/
 
 # cmake related
 CMakeFiles*

.travis.yml

@@ -0,0 +1,21 @@
+matrix:
+  include:
+  - os: osx
+    env: COMPILER='clang++'
+    osx_image: xcode8.2
+  - os: osx
+    env: COMPILER='clang++'
+    osx_image: xcode8
+
+install:
+  - if [ $TRAVIS_OS_NAME == osx ]; then brew update ; brew uninstall --force cmake; brew install cmake ; fi
+  - git submodule update --init --recursive
+
+before_script:
+  - export CXX=$COMPILER
+  - cd test
+  - mkdir darwin && cd darwin
+
+script:
+  - cmake .. -DTRAVIS=ON -DEXTRA_TESTS=ON
+  - make VERBOSE=1 && ./unittests --pass

CMakeLists.txt

@@ -2,39 +2,63 @@ cmake_minimum_required(VERSION 2.8.9)
 
 set(INCLUDEOS_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/)
 
-set(CMAKE_TOOLCHAIN_FILE ${INCLUDEOS_ROOT}/cmake/i686-elf-toolchain.cmake)
+# Target CPU Architecture
+if(DEFINED ENV{ARCH})
+  set(ARCH "$ENV{ARCH}" CACHE STRING "Architecture")
+else()
+  set(ARCH "x86_64" CACHE STRING "Architecture (default)")
+endif()
 
-project (includeos)
+message(STATUS "Target CPU ${ARCH}")
+
+set(TRIPLE "${ARCH}-pc-linux-elf")
+set(CMAKE_CXX_COMPILER_TARGET ${TRIPLE})
+set(CMAKE_C_COMPILER_TARGET ${TRIPLE})
+
+message(STATUS "Target triple ${TRIPLE}")
+
+set(CMAKE_TOOLCHAIN_FILE ${INCLUDEOS_ROOT}/cmake/${ARCH}-elf-toolchain.cmake)
+
+project (includeos C CXX)
 
 set(LIB ${CMAKE_INSTALL_PREFIX}/includeos/lib)
 set(BIN ${CMAKE_INSTALL_PREFIX}/includeos/bin)
 set(SCRIPTS ${CMAKE_INSTALL_PREFIX}/includeos/scripts)
 
-# test compiler
-if(CMAKE_COMPILER_IS_GNUCC)
-	# currently gcc is not supported due to problems cross-compiling a unikernel
-	# (i.e., building a 32bit unikernel (only supported for now) on a 64bit system)
-	message(FATAL_ERROR "Building IncludeOS with gcc is not currently supported. Please clean-up build directory and configure for clang through CC and CXX environmental variables.")
-endif(CMAKE_COMPILER_IS_GNUCC)
+# C++ version
+set(CMAKE_CXX_STANDARD 14)
 
 # create OS version string from git describe (used in CXX flags)
 execute_process(COMMAND git describe --dirty
 	        WORKING_DIRECTORY ${INCLUDEOS_ROOT}
 	        OUTPUT_VARIABLE OS_VERSION)
 string(STRIP ${OS_VERSION} OS_VERSION)
 
+option(cpu_feat_vanilla "Restrict use of CPU features to vanilla" ON)
+if(cpu_feat_vanilla)
+  include("cmake/vanilla.cmake")
+  set(DEFAULT_SETTINGS_CMAKE "vanilla.cmake") # for service cmake
+  set(DEFAULT_VM "vm.vanilla.json") # vmrunner
+else()
+  include("cmake/cpu_feat.cmake")
+  set(DEFAULT_SETTINGS_CMAKE "cpu_feat.cmake") # for service cmake
+  set(DEFAULT_VM "vm.cpu_feat.json") # vmrunner
+endif(cpu_feat_vanilla)
+
+option(single_threaded "Compile without SMP support" ON)
+
 # create random hex string as stack protector canary
 string(RANDOM LENGTH 8 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE)
 
-set(CAPABS "-msse3 -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE} ")
+set(CAPABS "${CAPABS} -mno-red-zone -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}")
 
 # Various global defines
 # * NO_DEBUG disables output from the debug macro
 # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures
 # * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h)
 set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE")
 
-set(WARNS "-Wall -Wextra") #-pedantic
+set(WARNS "-Wall -Wextra")
 
 # configure options
 option(debug "Build with debugging symbols (OBS: Dramatically increases binary size)" OFF)
@@ -52,12 +76,15 @@ endfunction()
 init_submodule(mod/GSL)
 init_submodule(mod/http-parser)
 init_submodule(mod/uzlib)
+init_submodule(mod/rapidjson)
+
+install(DIRECTORY mod/rapidjson/include/rapidjson DESTINATION includeos/include)
 
 # set optimization level
 set(OPTIMIZE "-O2")
 
 if(debug OR debug-info OR debug-all)
-	set(CAPABS "${CAPABS} -O0")
+	set(OTPIMIZE "-O0")
 endif(debug OR debug-info OR debug-all)
 
 if(minimal)
@@ -81,24 +108,41 @@ endif(silent)
 # Append optimization level
 set(CAPABS "${CAPABS} ${OPTIMIZE}")
 
-# these kinda work with llvm
-set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
-set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+if (CMAKE_COMPILER_IS_GNUCC)
+  # gcc/g++ settings
+  set(CMAKE_CXX_FLAGS " -MMD ${CAPABS} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
+  set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+else()
+  # these kinda work with llvm
+  set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c  -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
+  set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+endif()
 
 # either download or cross-compile needed libraries
-option(from_bundle "Download and use pre-compiled libraries for cross-comilation" ON)
+#option(from_bundle "Download and use pre-compiled libraries for cross-comilation" ON)
+set(BUNDLE_LOC "" CACHE STRING "Local path of bundle with pre-compile libraries")
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/cross_compiled_libraries.txt)
 
-add_subdirectory(mod)
+# Botan Crypto & TLS
+# Note: Include order matters!
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/botan.cmake)
 
+#
+# Subprojects
+#
+add_subdirectory(mod)
 add_subdirectory(src)
 
+#
+# External projects
+#
 option(vmbuild "Build and install vmbuild and elf_syms" ON)
 if(vmbuild)
   # Install vmbuilder as an external project
   ExternalProject_Add(vmbuild
     PREFIX vmbuild # Build where
     SOURCE_DIR ${INCLUDEOS_ROOT}/vmbuild # Where is project located
+    BINARY_DIR ${INCLUDEOS_ROOT}/vmbuild/build
     INSTALL_DIR ${BIN} # Where to install
     CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> # Pass installation folder
     )
@@ -128,27 +172,35 @@ if(tests)
   ExternalProject_Add(unittests
     PREFIX unittests
     SOURCE_DIR ${INCLUDEOS_ROOT}/test
+    BINARY_DIR unittests
     CMAKE_ARGS -DINCLUDEOS_ROOT=${INCLUDEOS_ROOT} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}
     )
   #add_subdirectory(test)
 endif(tests)
 
 option(libmana "Build and install mana web application framework library" ON)
 if(libmana)
-  set(rapidjson ON) # Dependent on rapidjson
   add_subdirectory(lib/mana)
 endif(libmana)
 
-option(rapidjson "Download and install rapidjson submodule" ON)
-if(rapidjson)
-  init_submodule(mod/rapidjson)
-  install(DIRECTORY mod/rapidjson/include/rapidjson DESTINATION includeos/include)
-endif(rapidjson)
+#
+# Installation
+#
+
+# Install cmake files
+install(FILES cmake/pre.service.cmake DESTINATION includeos)
+install(FILES cmake/post.service.cmake DESTINATION includeos)
+install(FILES cmake/library.cmake DESTINATION includeos)
+install(FILES cmake/${DEFAULT_SETTINGS_CMAKE} DESTINATION includeos RENAME settings.cmake) # cpu_feat_vanilla opt
+
+# Install vmrunner
+install(DIRECTORY vmrunner DESTINATION includeos)
+install(FILES vmrunner/${DEFAULT_VM} DESTINATION includeos/vmrunner/ RENAME vm.default.json) # cpu_feat_vanilla opt
+
+# Install toolchain
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${ARCH}-elf-toolchain.cmake DESTINATION includeos)
 
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/etc/service.cmake DESTINATION includeos)
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/etc/library.cmake DESTINATION includeos)
-install(DIRECTORY vmrunner DESTINATION includeos/)
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/i686-elf-toolchain.cmake DESTINATION includeos)
+# Install seed
 install(DIRECTORY seed/ DESTINATION includeos/seed)
 
 # Install boot util

README.md

@@ -22,15 +22,15 @@ IncludeOS is free software, with "no warranties or restrictions of any kind".
 
 ## Build status
 
-|        | Build from bundle                                                                                                                                             |
-|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Master | [![Build Status](https://jenkins.includeos.org/buildStatus/icon?job=shield_master_bundle)](https://jenkins.includeos.org/job/shield_master_bundle/) |
-| Dev    | [![Build Status](https://jenkins.includeos.org/buildStatus/icon?job=shield_dev_bundle)](https://jenkins.includeos.org/job/shield_dev_bundle/)      |
+|        | Build from bundle | Integration tests |
+|--------|-------------------|-------------------|
+| Master | [![Build Status](https://img.shields.io/jenkins/s/https/jenkins.includeos.org/shield_master_bundle.svg)](https://jenkins.includeos.org/job/shield_master_bundle/) |  Coming soon |
+| Dev    | [![Build Status](https://img.shields.io/jenkins/s/https/jenkins.includeos.org/shield_dev_bundle.svg)](https://jenkins.includeos.org/job/shield_dev_bundle/)      | [![Jenkins tests](https://img.shields.io/jenkins/t/https/jenkins.includeos.org/shield_dev_integration_tests.svg)](https://jenkins.includeos.org/job/shield_dev_integration_tests/) |
 
 ### Key features
 
 * **Extreme memory footprint**: A minimal bootable image, including bootloader, operating system components and a complete C++ standard library is currently 707K when optimized for size.
-* **KVM and VirtualBox support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on any modern x86 CPUs). In principle IncludeOS should run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), which power the [OpenStack IaaS cloud](https://www.openstack.org/), and [VirtualBox](https://www.virtualbox.org), which means that you can run your IncludeOS service on both Linux, Microsoft Windows and macOS.
+* **KVM, VirtualBox and VMWare support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on any modern x86 CPUs). In principle IncludeOS should run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), which power the [OpenStack IaaS cloud](https://www.openstack.org/), and [VirtualBox](https://www.virtualbox.org), which means that you can run your IncludeOS service on both Linux, Microsoft Windows and macOS.
 * **C++11/14 support**
     * Full C++11/14 language support with [clang](http://clang.llvm.org) v3.8 and later.
     * Standard C++ library (STL) [libc++](http://libcxx.llvm.org) from [LLVM](http://llvm.org/).

api/arch.hpp

@@ -0,0 +1,51 @@
+// -*-C++-*-
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#ifndef INCLUDEOS_ARCH_HEADER
+#define INCLUDEOS_ARCH_HEADER
+
+#include <cstddef>
+#include <cstdint>
+#include <cassert>
+
+extern void __arch_init();
+extern void __arch_poweroff();
+extern void __arch_reboot();
+extern void __arch_enable_legacy_irq(uint8_t);
+extern void __arch_disable_legacy_irq(uint8_t);
+inline void __arch_hw_barrier() noexcept;
+inline void __sw_barrier() noexcept;
+inline uint64_t __arch_cpu_cycles() noexcept;
+
+
+inline void __sw_barrier() noexcept
+{
+  asm volatile("" ::: "memory");
+}
+
+// Include arch specific inline implementations
+#if defined(ARCH_x86_64)
+#include "arch/x86_64.hpp"
+#elif defined(ARCH_i686)
+#include "arch/i686.hpp"
+#else
+#error "Unsupported arch specified"
+#endif
+
+#endif

api/arch/i686.hpp

@@ -0,0 +1,31 @@
+// -*-C++-*-
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef i686_ARCH_HPP
+#define i686_ARCH_HPP
+
+#include <arch/x86.hpp>
+
+inline uint64_t __arch_cpu_cycles() noexcept {
+  uint64_t ret;
+  asm("rdtsc" : "=A" (ret));
+  return ret;
+}
+
+
+#endif

api/arch/x86.hpp

@@ -0,0 +1,30 @@
+// -*-C++-*-
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef X86_ARCH_HPP
+#define X86_ARCH_HPP
+
+#define ARCH_x86
+
+
+inline void __arch_hw_barrier() noexcept
+{
+  asm volatile("mfence" ::: "memory");
+}
+
+#endif

api/arch/x86_64.hpp

@@ -0,0 +1,30 @@
+// -*-C++-*-
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef X86_64_ARCH_HPP
+#define X86_64_ARCH_HPP
+
+#include <arch/x86.hpp>
+
+inline uint64_t __arch_cpu_cycles() noexcept {
+  uint32_t hi, lo;
+  asm("rdtsc" : "=a"(lo), "=d"(hi));
+  return ((uint64_t) lo) | ((uint64_t) hi) << 32;
+}
+
+#endif