Merge branch 'master' into deploy

Author: mpe

README.md

@@ -69,6 +69,7 @@ There are images for various toolchains, they are encoded in the distro name/ver
  - `ubuntu@20.04`: Ubuntu 20.04, gcc 9.4.0, binutils 2.34
  - `ubuntu@18.04`: Ubuntu 18.04, gcc 7.5.0, binutils 2.30
  - `ubuntu@16.04`: Ubuntu 16.04, gcc 5.4.0, binutils 2.26.1
+ - `fedora@39`: Fedora 39, gcc 13.2.1, binutils 2.40-14, clang 17.0.6
  - `fedora@38`: Fedora 38, gcc 12.2.1, binutils 2.39-3, clang 16.0.2
  - `fedora@37`: Fedora 37, gcc 12.2.1, binutils 2.38-5, clang 15.0.7
  - `fedora@36`: Fedora 36, gcc 12.1.1, binutils 2.37-7, clang 14.0.5

bootstrap.sh

@@ -23,7 +23,7 @@ if [[ "$ID" == "fedora" ]]; then
         tar \
         xz
     )
-elif [[ "$ID_LIKE" == "debian" ]]; then
+elif [[ "${ID_LIKE:-$ID}" == "debian" ]]; then
     export DEBIAN_FRONTEND=noninteractive
     (set -x;
      $sudo apt-get -y install \

build/Makefile

@@ -1,6 +1,6 @@
 all: help
 
-export FEDORA_LATEST := 38
+export FEDORA_LATEST := 39
 
 x86 = $(filter x86_64,$(shell uname -m))
 isa_3 = $(filter arch_3_00,$(shell LD_SHOW_AUXV=1 /bin/true))
@@ -16,14 +16,14 @@ endif
 
 UBUNTU_DISTROS += ubuntu@20.04 ubuntu@18.04 ubuntu@16.04 ubuntu
 
-FEDORA_DISTROS := fedora@${FEDORA_LATEST} fedora@37 fedora@36 fedora@35 fedora@34 fedora@33 fedora@31 fedora
+FEDORA_DISTROS := fedora@${FEDORA_LATEST} fedora@38 fedora@37 fedora@36 fedora@35 fedora@34 fedora@33 fedora@31 fedora
 KORG_DISTROS := korg@13.1.0 korg@12.3.0 korg@12.2.0 korg@12.1.0 korg@11.3.0 korg@11.1.0 korg@10.3.0 korg@9.4.0 korg@9.3.0 korg@8.5.0 korg@8.1.0 korg@5.5.0
 ALL_DISTROS := ${UBUNTU_DISTROS} ${KORG_DISTROS} ${FEDORA_DISTROS}
 DOCS_DISTRO := docs@${UBUNTU_LATEST}
 X86_DISTROS := ubuntu@${UBUNTU_LATEST} ubuntu
-ALIAS_DISTROS := ubuntu ubuntu-allcross fedora
-ALLCROSS_DISTROS := ubuntu-allcross@${UBUNTU_LATEST} ubuntu-allcross
-ALL_ARCHES := alpha arm arm64 i686 m68k mips mips64 riscv s390 sh sparc x86_64
+ALIAS_DISTROS := ubuntu allcross fedora
+ALLCROSS_DISTROS := allcross@sid allcross
+ALL_ARCHES := alpha arc arm arm64 i686 m68k mips mips64 parisc riscv s390 sh sparc x86_64
 SUBARCHES := ppc64le ppc64 ppc
 
 VERSION:
@@ -32,7 +32,7 @@ VERSION:
 define MAIN_TEMPLATE =
 image@${1}@${2} rebuild-image@${1}@${2}: VERSION
 
-image@${1}@${2} rebuild-image@${1}@${2} pull-image@${1}@${2} push-image@${1}@${2}:
+image@${1}@${2} rebuild-image@${1}@${2} pull-image@${1}@${2} push-image@${1}@${2} pull-base-image@${1}@${2}:
 	@./scripts/image.sh $$@
 
 clean@${1}@${2}:
@@ -43,6 +43,7 @@ IMAGES += image@${1}@${2}
 PULL_IMAGES += pull-image@${1}@${2}
 REBUILD_IMAGES += rebuild-image@${1}@${2}
 PUSH_IMAGES += push-image@${1}@${2}
+PULL_BASE_IMAGES += pull-base-image@${1}@${2}
 endef
 
 define KERNEL_TEMPLATE =
@@ -52,6 +53,9 @@ kernel@${1}@${2}: image@${1}@${2}
 clean-kernel@${1}@${2}:
 	@./scripts/clean.sh $$@
 
+prune-kernel@${1}@${2}:
+	@./scripts/prune.sh $$@
+
 KERNEL += $(if $(filter-out ${ALIAS_DISTROS},${2}), kernel@${1}@${2})
 endef
 
@@ -62,6 +66,9 @@ ppctests@${1}@${2} selftests@${1}@${2}: image@${1}@${2}
 clean-selftests@${1}@${2}:
 	@./scripts/clean.sh $$@
 
+prune-selftests@${1}@${2}:
+	@./scripts/prune.sh $$@
+
 PPCTESTS += $(if $(filter-out ${ALIAS_DISTROS},${2}), ppctests@${1}@${2})
 SELFTESTS += $(if $(filter-out ${ALIAS_DISTROS},${2}), selftests@${1}@${2})
 endef
@@ -141,9 +148,10 @@ images: ${IMAGES}
 pull-images: ${PULL_IMAGES}
 rebuild-images: ${REBUILD_IMAGES}
 push-images: ${PUSH_IMAGES}
+pull-base-images: ${PULL_BASE_IMAGES}
 docs: ${DOCS}
 
-ALL_TARGETS = ${KERNEL} ${PPCTESTS} ${SELFTESTS} ${IMAGES} ${PULL_IMAGES} ${DOCS} ${CLEAN}
+ALL_TARGETS = ${KERNEL} ${PPCTESTS} ${SELFTESTS} ${PERF} ${IMAGES} ${PULL_IMAGES} ${REBUILD_IMAGES} ${PUSH_IMAGES} ${PULL_BASE_IMAGES} ${DOCS} ${CLEAN}
 .PHONY: ${ALL_TARGETS} VERSION
 
 empty:=

build/allcross/Dockerfile

@@ -0,0 +1,63 @@
+ARG from
+FROM ${from}
+ARG from
+
+RUN apt-get -q -y update && \
+    DEBIAN_FRONTEND=noninteractive \
+    apt-get -q -y install --no-install-recommends \
+      bc \
+      bison \
+      bsdmainutils \
+      bzip2 \
+      ca-certificates \
+      ccache \
+      cpio \
+      crossbuild-essential-powerpc \
+      gcc-multilib-powerpc-linux-gnu \
+      dwarves \
+      file \
+      flex \
+      gcc \
+      gcc-powerpc-linux-gnu g++-powerpc-linux-gnu \
+      gcc-aarch64-linux-gnu g++-aarch64-linux-gnu \
+      gcc-alpha-linux-gnu g++-alpha-linux-gnu \
+      gcc-arc-linux-gnu g++-arc-linux-gnu \
+      gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf \
+      gcc-hppa-linux-gnu g++-hppa-linux-gnu \
+      gcc-i686-linux-gnu g++-i686-linux-gnu \
+      gcc-m68k-linux-gnu g++-m68k-linux-gnu \
+      gcc-mips64el-linux-gnuabi64 g++-mips64el-linux-gnuabi64 \
+      gcc-mipsel-linux-gnu g++-mipsel-linux-gnu \
+      gcc-riscv64-linux-gnu g++-riscv64-linux-gnu \
+      gcc-s390x-linux-gnu g++-s390x-linux-gnu \
+      gcc-sh4-linux-gnu g++-sh4-linux-gnu \
+      gcc-sparc64-linux-gnu g++-sparc64-linux-gnu \
+      gcc-x86-64-linux-gnu g++-x86-64-linux-gnu \
+      git \
+      kmod \
+      libc-dev \
+      libelf-dev \
+      libfuse-dev \
+      liblz4-tool \
+      libssl-dev \
+      lzop \
+      make \
+      openssl \
+      python3 \
+      python3-dev \
+      u-boot-tools \
+      rename \
+      rsync \
+      sparse \
+      xz-utils && \
+    rm -rf /var/lib/apt/lists/* /tmp/packages.sh /var/cache/* /var/log/dpkg.log
+
+ARG uid
+ARG gid
+
+RUN groupadd --gid $gid linuxppc
+RUN useradd --uid $uid --gid $gid linuxppc
+USER linuxppc
+
+COPY scripts/container-build.sh /bin/container-build.sh
+COPY VERSION /VERSION

build/scripts/build.sh

@@ -25,10 +25,14 @@ alternate_binds=$(get_alternate_binds)
 arch=$subarch
 if [[ "$subarch" == "alpha" ]]; then
     cross="alpha-linux-gnu-"
+elif [[ "$subarch" == "arc" ]]; then
+    cross="arc-linux-gnu-"
 elif [[ "$subarch" == "arm" ]]; then
     cross="arm-linux-gnueabihf-"
 elif [[ "$subarch" == "arm64" ]]; then
     cross="aarch64-linux-gnu-"
+elif [[ "$subarch" == "parisc" ]]; then
+    cross="hppa-linux-gnu-"
 elif [[ "$subarch" == "i686" ]]; then
     cross="i686-linux-gnu-"
     arch=x86
@@ -135,7 +139,22 @@ fi
 
 if [[ "$task" == "kernel" ]]; then
     if [[ -z "$DEFCONFIG" ]]; then
-        DEFCONFIG="${subarch}_defconfig"
+        case "$subarch" in
+        alpha)  ;&
+        arm64)  ;&
+        riscv)  ;&
+        s390)   DEFCONFIG=defconfig ;;
+        arc)    DEFCONFIG=axs103_smp_defconfig ;;
+        arm)    DEFCONFIG=versatile_defconfig ;;
+        i686)   DEFCONFIG=i386_defconfig ;;
+        m68k)   DEFCONFIG=multi_defconfig ;;
+        mips)   DEFCONFIG=32r6_defconfig ;;
+        mips64) DEFCONFIG=64r6_defconfig ;;
+        parisc) DEFCONFIG=generic-32bit_defconfig ;;
+        sh)     DEFCONFIG=shx3_defconfig ;;
+        sparc)  DEFCONFIG=sparc64_defconfig ;;
+        *)      DEFCONFIG="${subarch}_defconfig" ;;
+        esac
     fi
     cmd+="-e DEFCONFIG=$DEFCONFIG "
 

build/scripts/clean.sh

@@ -13,4 +13,8 @@ IFS=@ read -r task subarch distro version <<< "$1"
 
 output_dir=$(get_output_dir "$script_base" "$subarch" "$distro" "$version" "$task" "$DEFCONFIG" "$TARGETS" "$CLANG" "")
 
-(set -x ; rm -rf "$output_dir")
+if [[ -n $QUIET ]]; then
+    rm -rf "$output_dir"
+else
+    (set -x ; rm -rf "$output_dir")
+fi

build/scripts/image.sh

@@ -17,6 +17,23 @@ fi
 
 image="linuxppc/build:$distro-$version"
 
+from="docker.io/$distro:$version"
+
+if [[ "$distro" == "docs" ]]; then
+    from="docker.io/ubuntu:$version"
+elif [[ "$distro" == "allcross" ]]; then
+    from="docker.io/debian:$version"
+elif [[ "$distro" == "korg" ]]; then
+    # Use an older distro for the 5.x toolchains.
+    if [[ "$version" == 5.* ]]; then
+        from="docker.io/ubuntu:16.04"
+    elif [[ "$version" == 13.* ]]; then
+        from="docker.io/ubuntu:23.04"
+    else
+        from="docker.io/ubuntu:20.04"
+    fi
+fi
+
 if [[ "$task" == "image" ]]; then
     cmd="$DOCKER images -q --filter=reference=$image"
     echo "+ $cmd" # fake set -x display
@@ -82,14 +99,14 @@ elif [[ "$task" == "push-image" ]]; then
     fi
 
     exit 0
+elif [[ "$task" == "pull-base-image" ]]; then
+    cmd="$DOCKER pull $from"
+    (set -x; $cmd)
+    exit $?
 fi
 
 cmd="$DOCKER build -f $distro/Dockerfile "
 
-if [[ "$distro" != "ubuntu-allcross" ]]; then
-    cmd+="--pull "
-fi
-
 if [[ -n "$http_proxy" ]]; then
     cmd+="--build-arg http_proxy=$http_proxy "
 fi
@@ -106,28 +123,11 @@ if [[ -z "$GID" ]]; then
     GID=$(id -g)
 fi
 
-from="docker.io/$distro:$version"
-
-if [[ "$distro" == "docs" ]]; then
-    from="docker.io/ubuntu:$version"
-elif [[ "$distro" == "ubuntu-allcross" ]]; then
-	from="linuxppc/build:ubuntu-$version"
-elif [[ "$distro" == "korg" ]]; then
+if [[ "$distro" == "korg" ]]; then
     cmd+="--build-arg compiler_version=$version "
-
     arch=$(uname -m)
-
     cmd+="--build-arg base_url=https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/${arch}/${version}/ "
     cmd+="--build-arg tar_file=${arch}-gcc-${version}-nolibc-powerpc64-linux.tar.xz "
-
-    # Use an older distro for the 5.x toolchains.
-    if [[ "$version" == 5.* ]]; then
-	from="docker.io/ubuntu:16.04"
-    elif [[ "$version" == 13.* ]]; then
-	from="docker.io/ubuntu:23.04"
-    else
-	from="docker.io/ubuntu:20.04"
-    fi
 fi
 
 cmd+="--build-arg uid=$UID "

build/scripts/lib.sh

@@ -41,6 +41,7 @@ function get_output_dir()
 
     case "$task" in
         kernel) ;&
+        prune-kernel) ;&
         clean-kernel)
 	    if [[ -n "$symlink" ]]; then
 		echo "$d/latest-kernel"
@@ -53,6 +54,7 @@ function get_output_dir()
             ;;
         ppctests) ;&
         selftests) ;&
+        prune-selftests) ;&
         clean-selftests)
 	    if [[ -n "$symlink" ]]; then
 		echo "$d/latest-selftests"
@@ -89,7 +91,7 @@ function get_output_dir()
         d="$d@$version"
     fi
 
-    d=$(realpath $d)
+    d=$(realpath -q $d)
     echo "$d"
 
     return 0
@@ -102,7 +104,7 @@ function get_default_version()
 
     case "$distro" in
         ubuntu) latest="$UBUNTU_LATEST" ;;
-        ubuntu-allcross) latest="$UBUNTU_LATEST" ;;
+        allcross) latest="sid" ;;
         fedora) latest="$FEDORA_LATEST" ;;
     esac
 

build/scripts/prune.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+if [[ -z "$1" ]]; then
+    echo "Usage: $0 <target>" >&2
+    exit 1
+fi
+
+dir="$(dirname "$0")"
+script_base="$(realpath "$dir")"
+. "$script_base/lib.sh"
+
+IFS=@ read -r task subarch distro version <<< "$1"
+
+output_dir=$(get_output_dir "$script_base" "$subarch" "$distro" "$version" "$task" "$DEFCONFIG" "$TARGETS" "$CLANG")
+
+case "$task" in
+    prune-kernel)
+        if [[ ! -e "$output_dir/Makefile" ]]; then
+            # Assume it's already been pruned
+            exit 0
+        fi
+
+        echo "Pruning non-outputs in $output_dir"
+
+        set -euo pipefail
+        cd "$output_dir"
+        mkdir -p "artifacts"
+        for path in .config vmlinux System.map arch/powerpc/boot/zImage include/config/kernel.release \
+                    arch/powerpc/kernel/asm-offsets.s arch/powerpc/boot/uImage modules.tar.bz2 \
+                    sparse.log log.txt
+        do
+            if [[ -e "$path" ]]; then
+                mv "$path" artifacts/
+            fi
+        done
+        find . -not -path "./artifacts*" -delete
+        mv artifacts/.config config
+        mv artifacts/* .
+        rmdir artifacts
+        ;;
+    prune-selftests)
+        if [[ ! -e "$output_dir/kselftest" ]]; then
+            # Assume it's already been pruned
+            exit 0
+        fi
+
+        echo "Pruning non-outputs in $output_dir"
+
+        set -euo pipefail
+        cd "$output_dir"
+        if [[ -d install ]]; then
+            mv install selftests
+            tar -czf selftests.tar.gz selftests
+        fi
+        find . -not -path "./selftests.tar.gz" -delete
+        ;;
+    *)
+        (set -x ; rm -rf "$output_dir")
+        ;;
+esac