build: fixes to bundle build using newlib 2.5

Author: alfreb

CMakeLists.txt

@@ -26,7 +26,7 @@ string(STRIP ${OS_VERSION} OS_VERSION)
 # create random hex string as stack protector canary
 string(RANDOM LENGTH 8 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE)
 
-set(CAPABS "-mavx -maes -mfma -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE} ")
+set(CAPABS "-msse3 -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE} ")
 
 # Various global defines
 # * NO_DEBUG disables output from the debug macro

api/posix/ctype.h

@@ -15,6 +15,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Trick to prevent libc++ from using old newlib locale stubs
+#define _LIBCPP_SUPPORT_NEWLIB_XLOCALE_H
+
+
 #ifndef SYS_CTYPE_H
 #define SYS_CTYPE_H
 
@@ -40,5 +44,4 @@ extern "C" {
 
 #endif //SYS_CTYPE_H
 
-
 #include_next <ctype.h>

api/posix/features.h

@@ -27,6 +27,11 @@
 #define _POSIX_TIMERS 1
 #define _POSIX_MONOTONIC_CLOCK 1
 
+// Enable newlib multibyte support
+#define _MB_CAPABLE 1
+
+
+
 // Required to pass CMake tests for libc++
 #define __GLIBC_PREREQ__(min, maj) 1
 #define __GLIBC_PREREQ(min, maj) 1

cmake/cross_compiled_libraries.txt

@@ -32,8 +32,12 @@ set(PRECOMPILED_DIR ${CMAKE_CURRENT_BINARY_DIR}/precompiled/src/PrecompiledLibra
 set(LIBCXX_INCLUDE_DIR ${PRECOMPILED_DIR}libcxx/include/)
 set(LIBCXX_LIB_DIR ${PRECOMPILED_DIR}/libcxx/)
 add_library(libcxx STATIC IMPORTED)
+add_library(libcxxabi STATIC IMPORTED)
+
 add_dependencies(libcxx PrecompiledLibraries)
+add_dependencies(libcxxabi PrecompiledLibraries)
 set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++.a)
+set_target_properties(libcxxabi PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++abi.a)
 
 set(NEWLIB_INCLUDE_DIR ${PRECOMPILED_DIR}/newlib/include/)
 set(NEWLIB_LIB_DIR ${PRECOMPILED_DIR}/newlib/)
@@ -62,4 +66,4 @@ install(DIRECTORY ${NEWLIB_INCLUDE_DIR} DESTINATION includeos/include/newlib)
 
 install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/lib)
 
-install(FILES ${NEWLIB_LIB_DIR}/libc.a ${NEWLIB_LIB_DIR}/libg.a ${NEWLIB_LIB_DIR}/libm.a ${LIBGCC_LIB_DIR}/libgcc.a ${LIBCXX_LIB_DIR}/libc++.a DESTINATION includeos/lib)
+install(FILES ${NEWLIB_LIB_DIR}/libc.a ${NEWLIB_LIB_DIR}/libg.a ${NEWLIB_LIB_DIR}/libm.a ${LIBGCC_LIB_DIR}/libgcc.a ${LIBCXX_LIB_DIR}/libc++.a ${LIBCXX_LIB_DIR}/libc++abi.a DESTINATION includeos/lib)

etc/build_llvm32.sh

@@ -5,7 +5,10 @@
 
 newlib_inc=$TEMP_INSTALL_DIR/i686-elf/include	# path for newlib headers
 IncludeOS_posix=$INCLUDEOS_SRC/api/posix
-libcxx_inc=$BUILD_DIR/$llvm_src/projects/libcxx/include
+libcxx_inc=$BUILD_DIR/llvm/projects/libcxx/include
+libcxxabi_inc=$BUILD_DIR/llvm/projects/libcxxabi/include
+
+# sysroot=$BUILD_DIR/IncludeOS_TEMP_install/i686-elf
 
 # Install dependencies
 sudo apt-get install -y cmake ninja-build subversion zlib1g-dev libtinfo-dev
@@ -19,7 +22,7 @@ if [ ! -z $download_llvm ]; then
     svn co http://llvm.org/svn/llvm-project/llvm/tags/$LLVM_TAG llvm
 
     # Clone libc++, libc++abi, and some extra stuff (recommended / required for clang)
-    cd llvm/projects
+    pushd llvm/projects
 
     # Compiler-rt
     svn co http://llvm.org/svn/llvm-project/compiler-rt/tags/$LLVM_TAG compiler-rt
@@ -38,7 +41,7 @@ if [ ! -z $download_llvm ]; then
     #git clone http://llvm.org/git/libunwind
 
     # Back to start
-	cd $BUILD_DIR
+    popd
 fi
 
 # Make a build-directory
@@ -49,54 +52,39 @@ if [ ! -z $clear_llvm_build_cache ]; then
     rm CMakeCache.txt
 fi
 
-# General options
-OPTS=-DCMAKE_EXPORT_COMPILE_COMMANDS=ON" "
-
-# LLVM General options
-OPTS+=-DBUILD_SHARED_LIBS=OFF" "
-OPTS+=-DCMAKE_BUILD_TYPE=MinSizeRel" "
-
-# Can't build libc++ with g++ unless it's a cross compiler (need to specify target)
-OPTS+=-DCMAKE_C_COMPILER=clang-$clang_version" "
-OPTS+=-DCMAKE_CXX_COMPILER=clang++-$clang_version" " # -std=c++11" "
-
 TRIPLE=i686-pc-none-elf
-
-OPTS+=-DTARGET_TRIPLE=$TRIPLE" "
-OPTS+=-DLLVM_BUILD_32_BITS=ON" "
-OPTS+=-DLLVM_INCLUDE_TESTS=OFF" "
-OPTS+=-DLLVM_ENABLE_THREADS=OFF" "
-OPTS+=-DLLVM_DEFAULT_TARGET_TRIPLE=$TRIPLE" "
-
-# libc++-specific options
-OPTS+=-DLIBCXX_ENABLE_SHARED=OFF" "
-OPTS+=-DLIBCXX_ENABLE_THREADS=OFF" "
-OPTS+=-DLIBCXX_TARGET_TRIPLE=$TRIPLE" "
-OPTS+=-DLIBCXX_BUILD_32_BITS=ON" "
-
-OPTS+=-DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON" "
-
-OPTS+=-DLIBCXX_CXX_ABI=libcxxabi" "
-OPTS+=-DLIBCXX_CXX_ABI_INCLUDE_PATHS=$INCLUDEOS_SRC/src/include" "
-
-# libunwind-specific options
-OPTS+=-DLIBUNWIND_ENABLE_SHARED=OFF" "
-OPTS+=-LIBCXXABI_USE_LLVM_UNWINDER=ON" "
-
-echo "LLVM CMake Build options:" $OPTS
-
-# CXX flags
-CXX_FLAGS="-std=c++14 -nostdlibinc -mavx -maes -mfma"
+CXX_FLAGS="-std=c++14 -msse3"
 
 # CMAKE configure step
 #
 # Include-path ordering:
 # 1. IncludeOS_posix has to come first, as it provides lots of C11 prototypes that libc++ relies on, but which newlib does not provide (see our math.h)
 # 2. libcxx_inc must come before newlib, due to math.h function wrappers around C99 macros (signbit, nan etc)
 # 3. newlib_inc provodes standard C headers
-cmake -GNinja $OPTS  -DCMAKE_CXX_FLAGS="$CXX_FLAGS -I$IncludeOS_posix -I$libcxx_inc -I$newlib_inc" $BUILD_DIR/llvm
+cmake -GNinja $OPTS  \
+      -DCMAKE_CXX_FLAGS="$CXX_FLAGS -I$IncludeOS_posix -I$libcxxabi_inc -I$libcxx_inc -I$newlib_inc " $BUILD_DIR/llvm \
+      -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+      -DBUILD_SHARED_LIBS=OFF \
+      -DCMAKE_C_COMPILER=clang-$clang_version \
+      -DCMAKE_CXX_COMPILER=clang++-$clang_version \
+      -DTARGET_TRIPLE=$TRIPLE \
+      -DLLVM_BUILD_32_BITS=ON \
+      -DLLVM_INCLUDE_TESTS=OFF \
+      -DLLVM_ENABLE_THREADS=OFF \
+      -DLLVM_DEFAULT_TARGET_TRIPLE=$TRIPLE \
+      -DLIBCXX_ENABLE_SHARED=OFF \
+      -DLIBCXX_ENABLE_THREADS=OFF \
+      -DLIBCXX_TARGET_TRIPLE=$TRIPLE \
+      -DLIBCXX_BUILD_32_BITS=ON \
+      -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
+      -DLIBCXX_CXX_ABI=libcxxabi \
+      -DLIBCXXABI_TARGET_TRIPLE=$TRIPLE \
+      -DLIBCXXABI_ENABLE_THREADS=OFF \
+      -DLIBCXXABI_HAS_PTHREAD_API=OFF
+
 
 # MAKE
+ninja libc++abi.a
 ninja libc++.a
 
 popd

etc/build_newlib.sh

@@ -32,10 +32,18 @@ pushd build_newlib
 	--target=$TARGET \
 	--prefix=$TEMP_INSTALL_DIR \
 	--enable-newlib-io-long-long AS_FOR_TARGET=as LD_FOR_TARGET=ld AR_FOR_TARGET=ar RANLIB_FOR_TARGET=ranlib \
-
-echo -e "\n\n >>> BUILDING NEWLIB \n\n"    
+  --enable-newlib-hw-fp \
+  --enable-newlib-mb \
+  --enable-newlib-iconv \
+  --enable-newlib-iconv-encodings=utf-16,utf-8,ucs_2 \
+  --disable-libgloss \
+  --disable-multilib \
+  --enable-newlib-multithread \
+  --disable-newlib-supplied-syscalls
+
+echo -e "\n\n >>> BUILDING NEWLIB \n\n"
 # Compile
-make $num_jobs all 
+make $num_jobs all
 # Install
 make install
 

etc/create_binary_bundle.sh

@@ -9,13 +9,14 @@ export PATH="$TEMP_INSTALL_DIR/bin:$PATH"
 
 # Build options
 export TARGET=i686-elf	# Configure target
-export num_jobs=${num_jobs:-"-j"}	# Specify number of build jobs	
+export num_jobs=${num_jobs:-"-j"}	# Specify number of build jobs
 
 # Version numbers
-export binutils_version=${binutils_version:-2.26}		# ftp://ftp.gnu.org/gnu/binutils
-export newlib_version=${newlib_version:-2.4.0}			# ftp://sourceware.org/pub/newlib
-export gcc_version=${gcc_version:-6.2.0}				# ftp://ftp.nluug.nl/mirror/languages/gcc/releases/
-export clang_version=${clang_version:-3.8}				# http://releases.llvm.org/
+export binutils_version=${binutils_version:-2.28}		# ftp://ftp.gnu.org/gnu/binutils
+export newlib_version=${newlib_version:-2.5.0.20170323}			# ftp://sourceware.org/pub/newlib
+#export newlib_version=${newlib_version:-2.5.0}			# ftp://sourceware.org/pub/newlib
+export gcc_version=${gcc_version:-6.3.0}				# ftp://ftp.nluug.nl/mirror/languages/gcc/releases/
+export clang_version=${clang_version:-3.9}				# http://releases.llvm.org/
 export LLVM_TAG=${LLVM_TAG:-RELEASE_381/final}			# http://llvm.org/svn/llvm-project/llvm/tags
 
 # Options to skip steps
@@ -50,8 +51,11 @@ DEPS_BUILD="build-essential make nasm texinfo clang-$clang_version clang++-$clan
 
 echo -e "\n\n >>> Trying to install prerequisites for *building* IncludeOS"
 echo -e  "        Packages: $DEPS_BUILD \n"
-sudo apt-get update
-sudo apt-get install -y $DEPS_BUILD
+
+if [ ! -z $do_packages ]; then
+  sudo apt-get update
+  sudo apt-get install -y $DEPS_BUILD
+fi
 
 # Print currently set install options
 printf "\n\n>>> Bundle will be created with the following options:\n\n"
@@ -113,10 +117,10 @@ filename_tag=`echo $tag | tr . -`
 popd
 
 # Where to place the installation bundle
-DIR_NAME="IncludeOS_install"
-export INSTALL_DIR=${INSTALL_DIR:-~/$DIR_NAME}
+DIR_NAME="IncludeOS_dependencies"
+export BUNDLE_DIR=${BUNDLE_DIR:-~/$DIR_NAME}
 
-echo ">>> Creating Installation Bundle as $INSTALL_DIR"
+echo ">>> Creating Installation Bundle as $BUNDLE_DIR"
 
 OUTFILE="${DIR_NAME}_$filename_tag.tar.gz"
 
@@ -128,6 +132,7 @@ libc=$newlib/libc.a
 libm=$newlib/libm.a
 libg=$newlib/libg.a
 libcpp=$llvm/lib/libc++.a
+libcppabi=$llvm/lib/libc++abi.a
 
 GPP=$TEMP_INSTALL_DIR/bin/i686-elf-g++
 GCC_VER=`$GPP -dumpversion`
@@ -138,26 +143,27 @@ include_newlib=$TEMP_INSTALL_DIR/i686-elf/include
 include_libcxx=$llvm/include/c++/v1
 
 # Make directory-tree
-mkdir -p $INSTALL_DIR
-mkdir -p $INSTALL_DIR/newlib
-mkdir -p $INSTALL_DIR/libcxx
-mkdir -p $INSTALL_DIR/crt
-mkdir -p $INSTALL_DIR/libgcc
+mkdir -p $BUNDLE_DIR
+mkdir -p $BUNDLE_DIR/newlib
+mkdir -p $BUNDLE_DIR/libcxx
+mkdir -p $BUNDLE_DIR/crt
+mkdir -p $BUNDLE_DIR/libgcc
 
 # Copy binaries
-cp $libcpp $INSTALL_DIR/libcxx/
-cp $libm $INSTALL_DIR/newlib/
-cp $libc $INSTALL_DIR/newlib/
-cp $libg $INSTALL_DIR/newlib/
-cp $libgcc $INSTALL_DIR/libgcc/
-cp $TEMP_INSTALL_DIR/lib/gcc/i686-elf/$GCC_VER/crt*.o $INSTALL_DIR/crt/
+cp $libcpp $BUNDLE_DIR/libcxx/
+cp $libcppabi $BUNDLE_DIR/libcxx/
+cp $libm $BUNDLE_DIR/newlib/
+cp $libc $BUNDLE_DIR/newlib/
+cp $libg $BUNDLE_DIR/newlib/
+cp $libgcc $BUNDLE_DIR/libgcc/
+cp $TEMP_INSTALL_DIR/lib/gcc/i686-elf/$GCC_VER/crt*.o $BUNDLE_DIR/crt/
 
 # Copy includes
-cp -r $include_newlib $INSTALL_DIR/newlib/
-cp -r $include_libcxx $INSTALL_DIR/libcxx/include
+cp -r $include_newlib $BUNDLE_DIR/newlib/
+cp -r $include_libcxx $BUNDLE_DIR/libcxx/include
 
 # Zip it
-tar -czvf $OUTFILE --directory=$INSTALL_DIR/../ $DIR_NAME
+tar -czvf $OUTFILE --directory=$BUNDLE_DIR/../ $DIR_NAME
 
 echo ">>> IncludeOS Installation Bundle created as $INSTALL_DIR and gzipped into $OUTFILE"
 

etc/install_from_bundle.sh

@@ -15,6 +15,7 @@ set -e
 
 INCLUDEOS_SRC=${INCLUDEOS_SRC-$HOME/IncludeOS}
 INCLUDEOS_PREFIX=${INCLUDEOS_PREFIX-/usr/local}
+num_jobs=${num_jobs-"-j 4"}
 
 # Try to find suitable compiler
 cc_list="clang-3.9 clang-3.8 clang-3.7 clang-3.6 clang"
@@ -54,7 +55,7 @@ cmake $INCLUDEOS_SRC \
 	  -Dtests=$INCLUDEOS_ENABLE_TEST \
 	  -DBUNDLE_LOC=$BUNDLE_LOC
 make PrecompiledLibraries
-make -j 4
+make $num_jobs
 make install
 popd
 

etc/scripts/qemu_cmd.sh

@@ -3,7 +3,6 @@ if [ "$(egrep -m 1 '^flags.*(vmx|svm)' /proc/cpuinfo)" ]
 then
     echo ">>> KVM: ON "
     export QEMU="qemu-system-x86_64 --enable-kvm"
-    export CPU="-cpu host"
 else
     echo ">>> KVM: OFF "
     export QEMU="qemu-system-i386"

etc/service.cmake

@@ -25,7 +25,7 @@ endif(CMAKE_COMPILER_IS_GNUCC)
 set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf")
 enable_language(ASM_NASM)
 
-set(CAPABS "-mavx -maes -mfma -fstack-protector-strong")
+set(CAPABS "-msse3 -fstack-protector-strong")
 
 # Various global defines
 # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures