Merge branch 'master' into deploy

Author: mpe

Makefile

@@ -1,5 +1,19 @@
 help:
 	@echo "See README.md for help"
 
-prep:
-	@$(MAKE) -C root-disks
+SUBDIRS := root-disks external tests
+
+$(SUBDIRS):
+	@$(MAKE) --no-print-directory -C $@ $(MAKECMDGOALS) 2>&1 | sed 's/^/$@: /'
+
+download: $(SUBDIRS)
+
+prepare: $(SUBDIRS)
+
+build: external
+
+clean: $(SUBDIRS)
+
+distclean: $(SUBDIRS)
+
+.PHONY: download prepare build clean distclean help $(SUBDIRS)

README.md

@@ -20,13 +20,13 @@ Clone this repo.
 ```
 $ cd ci-scripts
 $ cd build
-$ make pull-image@ppc64le@ubuntu@21.10
-$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.10 JFACTOR=$(nproc)
+$ make pull-image@ppc64le@ubuntu
+$ make SRC=~/src/linux kernel@ppc64le@ubuntu JFACTOR=$(nproc)
 ```
 
-This will build you a `ppc64le_defconfig` using the Ubuntu 21.10 toolchain.
+This will build you a `ppc64le_defconfig` using the latest Ubuntu toolchain.
 
-The kernel will be in `output/ppc64le@ubuntu@21.10/ppc64le_defconfig/vmlinux`.
+The kernel will be in `output/ppc64le@ubuntu/ppc64le_defconfig/vmlinux`.
 
 For more help try `make help`.
 
@@ -36,14 +36,14 @@ Building different defconfigs
 You can specify a defconfig with `DEFCONFIG`.
 
 ```
-$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.10 DEFCONFIG=powernv_defconfig JFACTOR=$(nproc)
+$ make SRC=~/src/linux kernel@ppc64le@ubuntu DEFCONFIG=powernv_defconfig JFACTOR=$(nproc)
 ```
 
 Note that the subarch (eg. `ppc64le`) needs to match the defconfig, so to build
 `ppc64_defconfig`, use `ppc64`.
 
 ```
-$ make SRC=~/src/linux kernel@ppc64@ubuntu@21.10 DEFCONFIG=ppc64_defconfig JFACTOR=$(nproc)
+$ make SRC=~/src/linux kernel@ppc64@ubuntu DEFCONFIG=ppc64_defconfig JFACTOR=$(nproc)
 ```
 
 Different toolchains
@@ -56,9 +56,9 @@ There are images for various toolchains, they are encoded in the distro name/ver
  - kernel.org gcc 9.3.0 `korg@9.3.0`
  - kernel.org gcc 8.1.0 `korg@8.1.0`
  - kernel.org gcc 5.5.0 `korg@5.5.0`
+ - Ubuntu 22.04 `ubuntu@22.04`
  - Ubuntu 21.10 `ubuntu@21.10`
  - Ubuntu 21.04 `ubuntu@21.04`
- - Ubuntu 20.10 `ubuntu@20.10`
  - Ubuntu 20.04 `ubuntu@20.04`
  - Ubuntu 18.04 `ubuntu@18.04`
  - Ubuntu 16.04 `ubuntu@16.04`
@@ -75,13 +75,13 @@ Building selftests
 To build the kernel selftests:
 
 ```
-$ make SRC=~/src/linux selftests@ppc64le@ubuntu@21.10 JFACTOR=$(nproc)
+$ make SRC=~/src/linux selftests@ppc64le@ubuntu JFACTOR=$(nproc)
 ```
 
 Or just the powerpc selftests:
 
 ```
-$ make SRC=~/src/linux ppctests@ppc64le@ubuntu@21.10 JFACTOR=$(nproc)
+$ make SRC=~/src/linux ppctests@ppc64le@ubuntu JFACTOR=$(nproc)
 ```
 
 You can also build the powerpc selftests with all available toolchains using:
@@ -95,17 +95,21 @@ Other options
 
 As mentioned above you pass the make -j factor with `JFACTOR=n`.
 
-To run sparse use the `ubuntu@21.10` image and pass `SPARSE=1`.
+To run sparse use the `ubuntu` image and pass `SPARSE=2`.
 
 ```
-$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.10 SPARSE=1 JFACTOR=$(nproc)
+$ make SRC=~/src/linux kernel@ppc64le@ubuntu SPARSE=2 JFACTOR=$(nproc)
 ```
 
-The log will be in eg. `output/ppc64le@ubuntu@21.10/ppc64le_defconfig/sparse.log`.
+The log will be in eg. `output/ppc64le@ubuntu/ppc64le_defconfig/sparse.log`.
+
+To only run sparse on files being recompiled, pass `SPARSE=1`.
 
 To build modules pass `MODULES=1`
 
-To build with clang pass `CLANG=1`, only works using the Ubuntu `21.10` images.
+To convert all modules to builtin, pass `MOD2YES=1`.
+
+To build with clang pass `CLANG=1`, only works using the latest Ubuntu image.
 
 For a quiet build pass `QUIET=1`, for verbose pass `VERBOSE=1`.
 
@@ -127,7 +131,7 @@ Building your own image
 If you don't want to pull an untrusted image, you can build it yourself with:
 
 ```
-$ make rebuild-image@ppc64le@ubuntu@21.10
+$ make rebuild-image@ppc64le@ubuntu
 ```
 
 Note that the build mounts the source tree read-only, so nothing it does can

bootstrap.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+
+. /etc/os-release
+
+echo "########################################"
+echo "# Installing basic packages ..."
+echo "########################################"
+
+sudo=""
+if [[ $(id -u) != 0 ]]; then
+    sudo="sudo"
+fi
+
+if [[ "$ID" == "fedora" ]]; then
+    (set -x;
+     $sudo dnf -y install \
+        expect \
+        make \
+        wget \
+        python3-pexpect \
+        tar \
+        xz
+    )
+elif [[ "$ID_LIKE" == "debian" ]]; then
+    export DEBIAN_FRONTEND=noninteractive
+    (set -x;
+     $sudo apt-get -y install \
+        expect \
+        make \
+        wget \
+        python3-pexpect \
+        python-is-python3 \
+        tar \
+        xz-utils
+    )
+else
+    echo "Unsupported distro!" >&2
+    exit 1
+fi
+
+nproc=$(nproc)
+
+echo "########################################"
+echo "# Downloading external dependencies ..."
+echo "########################################"
+make -j $nproc download
+
+echo "########################################"
+echo "# Installing additional packages ..."
+echo "########################################"
+
+# No -j because apt/dnf don't like being run in parallel
+make prepare
+
+echo "########################################"
+echo "# Building tools ..."
+echo "########################################"
+make -j $nproc build
+
+echo "########################################"
+echo "# Bootstrap completed"
+echo "########################################"

build/Makefile

@@ -1,5 +1,27 @@
 all: help
 
+export FEDORA_LATEST := 35
+export UBUNTU_LATEST := 22.04
+
+# If we're on ppc64le
+ifeq ($(shell uname -m),ppc64le)
+    # And we don't see ISA v3.0
+    ifeq ($(shell LD_SHOW_AUXV=1 /bin/true | grep "AT_HWCAP2:.*\barch_3_00\b"),)
+        # Then we can't use 22.04
+        UBUNTU_LATEST := 21.10
+    endif
+endif
+
+FEDORA_DISTROS := fedora@${FEDORA_LATEST} fedora@34 fedora@33 fedora@31 fedora
+# Sort is to elimiate duplicates if UBUNTU_LATEST == 21.10
+UBUNTU_DISTROS := $(sort ubuntu@${UBUNTU_LATEST} ubuntu@21.10 ubuntu@21.04 ubuntu@20.04 ubuntu@18.04 ubuntu@16.04 ubuntu)
+KORG_DISTROS := korg@11.1.0 korg@10.3.0 korg@9.3.0 korg@8.1.0 korg@5.5.0
+ALL_DISTROS := ${UBUNTU_DISTROS} ${KORG_DISTROS} ${FEDORA_DISTROS}
+DOCS_DISTRO := docs@${UBUNTU_LATEST}
+X86_DISTRO := ubuntu@${UBUNTU_LATEST}
+ALIAS_DISTROS := ubuntu fedora
+SUBARCHES := ppc64le ppc64 ppc
+
 VERSION:
 	@(git describe --always --long --dirty --abbrev=40) > VERSION
 
@@ -26,7 +48,7 @@ kernel@${1}@${2}: image@${1}@${2}
 clean-kernel@${1}@${2}:
 	@./scripts/clean.sh $$@
 
-KERNEL += kernel@${1}@${2}
+KERNEL += $(if $(filter-out ${ALIAS_DISTROS},${2}), kernel@${1}@${2})
 endef
 
 define SELFTESTS_TEMPLATE =
@@ -36,8 +58,8 @@ ppctests@${1}@${2} selftests@${1}@${2}: image@${1}@${2}
 clean-selftests@${1}@${2}:
 	@./scripts/clean.sh $$@
 
-PPCTESTS += ppctests@${1}@${2}
-SELFTESTS += selftests@${1}@${2}
+PPCTESTS += $(if $(filter-out ${ALIAS_DISTROS},${2}), ppctests@${1}@${2})
+SELFTESTS += $(if $(filter-out ${ALIAS_DISTROS},${2}), selftests@${1}@${2})
 endef
 
 define PERF_TEMPLATE =
@@ -47,7 +69,7 @@ perf@${1}@${2}: image@${1}@${2}
 clean-perf@${1}@${2}:
 	@./scripts/clean.sh $$@
 
-PERF += perf@${1}@${2}
+PERF += $(if $(filter-out ${ALIAS_DISTROS},${2}), perf@${1}@${2})
 endef
 
 define DOCS_TEMPLATE =
@@ -57,14 +79,6 @@ docs@${1}@${2}: image@${1}@${2}
 DOCS += docs@${1}@${2}
 endef
 
-FEDORA_DISTROS := fedora@35 fedora@34 fedora@33 fedora@31
-UBUNTU_DISTROS := ubuntu@21.10 ubuntu@21.04 ubuntu@20.10 ubuntu@20.04 ubuntu@18.04 ubuntu@16.04
-KORG_DISTROS := korg@11.1.0 korg@10.3.0 korg@9.3.0 korg@8.1.0 korg@5.5.0
-ALL_DISTROS := ${UBUNTU_DISTROS} ${KORG_DISTROS} ${FEDORA_DISTROS}
-DOCS_DISTRO := docs@21.10
-X86_DISTRO := ubuntu@21.10
-SUBARCHES := ppc64le ppc64
-
 $(eval $(call MAIN_TEMPLATE,ppc64le,${DOCS_DISTRO}))
 $(eval $(call DOCS_TEMPLATE,ppc64le,${DOCS_DISTRO}))
 

build/fedora/Dockerfile

@@ -8,6 +8,7 @@ RUN dnf -y install \
         ccache \
         cpio \
         diffutils \
+        dwarves \
         elfutils-libelf-devel \
         findutils \
         flex \

build/korg/Dockerfile

@@ -15,6 +15,7 @@ RUN apt-get -q -y update && \
       ccache \
       cpio \
       curl \
+      dwarves \
       file \
       flex \
       gcc \

build/scripts/build.sh

@@ -17,8 +17,6 @@ script_base="$(realpath "$dir")"
 
 IFS=@ read -r task subarch distro version <<< "$1"
 
-image="linuxppc/build:$distro-$version"
-
 SRC="${SRC/#\~/$HOME}"
 SRC=$(realpath "$SRC")
 
@@ -105,6 +103,10 @@ if [[ "$task" == "kernel" ]]; then
 	cmd+="-e MERGE_CONFIG=$MERGE_CONFIG "
     fi
 
+    if [[ -n "$MOD2YES" ]]; then
+	cmd+="-e MOD2YES=1 "
+    fi
+
     if [[ -n "$CLANG" ]]; then
         cmd+="-e CLANG=1 "
     fi
@@ -144,6 +146,13 @@ fi
 
 cmd+="$PODMAN_OPTS "
 
+if [[ -z "$version" ]]; then
+    # NB, after we passed $version to get_output_dir()
+    version=$(get_default_version $distro)
+fi
+
+image="docker.io/linuxppc/build:$distro-$version"
+
 cmd+="$image "
 cmd+="/bin/container-build.sh $task"
 

build/scripts/container-build.sh

@@ -78,11 +78,15 @@ if [[ "$1" == "kernel" ]]; then
 
     rc=$?
 
+    if [[ -n "$MOD2YES" ]]; then
+        (set -x; make $verbose $quiet "$cc" mod2yesconfig)
+    fi
+
     if [[ $rc -eq 0 ]]; then
         if [[ -n "$SPARSE" ]]; then
             rm -f /output/sparse.log
             touch /output/sparse.log
-            (set -x; make C=2 CF=">> /output/sparse.log 2>&1" $verbose $quiet "$cc" -j $JFACTOR)
+            (set -x; make C=$SPARSE CF=">> /output/sparse.log 2>&1" $verbose $quiet "$cc" -j $JFACTOR)
 
             rc=$?
 

build/scripts/image.sh

@@ -11,7 +11,11 @@ script_base="$(realpath "$dir")"
 
 IFS=@ read -r task subarch distro version <<< "$1"
 
-image="linuxppc/build:$distro-$version"
+if [[ -z "$version" ]]; then
+    version=$(get_default_version $distro)
+fi
+
+image="docker.io/linuxppc/build:$distro-$version"
 
 if [[ "$task" == "image" ]]; then
     cmd="$DOCKER images -q --filter=reference=$image"
@@ -64,10 +68,10 @@ if [[ -z "$GID" ]]; then
     GID=$(id -g)
 fi
 
-from="$distro:$version"
+from="docker.io/$distro:$version"
 
 if [[ "$distro" == "docs" ]]; then
-    from="ubuntu:$version"
+    from="docker.io/ubuntu:$version"
 elif [[ "$distro" == "korg" ]]; then
     cmd+="--build-arg compiler_version=$version "
 
@@ -78,9 +82,9 @@ elif [[ "$distro" == "korg" ]]; then
 
     # Use an older distro for the 5.x toolchains.
     if [[ "$version" == 5.* ]]; then
-	from="ubuntu:16.04"
+	from="docker.io/ubuntu:16.04"
     else
-	from="ubuntu:20.04"
+	from="docker.io/ubuntu:20.04"
     fi
 fi