Merge branch 'master' into deploy

Author: mpe

README.md

@@ -20,13 +20,13 @@ Clone this repo.
 ```
 $ cd ci-scripts
 $ cd build
-$ make pull-image@ppc64le@ubuntu@21.04
-$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.04 JFACTOR=$(nproc)
+$ make pull-image@ppc64le@ubuntu@21.10
+$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.10 JFACTOR=$(nproc)
 ```
 
-This will build you a `ppc64le_defconfig` using the Ubuntu 21.04 toolchain.
+This will build you a `ppc64le_defconfig` using the Ubuntu 21.10 toolchain.
 
-The kernel will be in `output/ppc64le@ubuntu@21.04/ppc64le_defconfig/vmlinux`.
+The kernel will be in `output/ppc64le@ubuntu@21.10/ppc64le_defconfig/vmlinux`.
 
 For more help try `make help`.
 
@@ -36,14 +36,14 @@ Building different defconfigs
 You can specify a defconfig with `DEFCONFIG`.
 
 ```
-$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.04 DEFCONFIG=powernv_defconfig JFACTOR=$(nproc)
+$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.10 DEFCONFIG=powernv_defconfig JFACTOR=$(nproc)
 ```
 
 Note that the subarch (eg. `ppc64le`) needs to match the defconfig, so to build
 `ppc64_defconfig`, use `ppc64`.
 
 ```
-$ make SRC=~/src/linux kernel@ppc64@ubuntu@21.04 DEFCONFIG=ppc64_defconfig JFACTOR=$(nproc)
+$ make SRC=~/src/linux kernel@ppc64@ubuntu@21.10 DEFCONFIG=ppc64_defconfig JFACTOR=$(nproc)
 ```
 
 Different toolchains
@@ -56,6 +56,7 @@ There are images for various toolchains, they are encoded in the distro name/ver
  - kernel.org gcc 9.3.0 `korg@9.3.0`
  - kernel.org gcc 8.1.0 `korg@8.1.0`
  - kernel.org gcc 5.5.0 `korg@5.5.0`
+ - Ubuntu 21.10 `ubuntu@21.10`
  - Ubuntu 21.04 `ubuntu@21.04`
  - Ubuntu 20.10 `ubuntu@20.10`
  - Ubuntu 20.04 `ubuntu@20.04`
@@ -74,13 +75,13 @@ Building selftests
 To build the kernel selftests:
 
 ```
-$ make SRC=~/src/linux selftests@ppc64le@ubuntu@21.04 JFACTOR=$(nproc)
+$ make SRC=~/src/linux selftests@ppc64le@ubuntu@21.10 JFACTOR=$(nproc)
 ```
 
 Or just the powerpc selftests:
 
 ```
-$ make SRC=~/src/linux ppctests@ppc64le@ubuntu@21.04 JFACTOR=$(nproc)
+$ make SRC=~/src/linux ppctests@ppc64le@ubuntu@21.10 JFACTOR=$(nproc)
 ```
 
 You can also build the powerpc selftests with all available toolchains using:
@@ -94,17 +95,17 @@ Other options
 
 As mentioned above you pass the make -j factor with `JFACTOR=n`.
 
-To run sparse use the `ubuntu@21.04` image and pass `SPARSE=1`.
+To run sparse use the `ubuntu@21.10` image and pass `SPARSE=1`.
 
 ```
-$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.04 SPARSE=1 JFACTOR=$(nproc)
+$ make SRC=~/src/linux kernel@ppc64le@ubuntu@21.10 SPARSE=1 JFACTOR=$(nproc)
 ```
 
-The log will be in eg. `output/ppc64le@ubuntu@21.04/ppc64le_defconfig/sparse.log`.
+The log will be in eg. `output/ppc64le@ubuntu@21.10/ppc64le_defconfig/sparse.log`.
 
 To build modules pass `MODULES=1`
 
-To build with clang pass `CLANG=1`, only works using the Ubuntu `21.04` images.
+To build with clang pass `CLANG=1`, only works using the Ubuntu `21.10` images.
 
 For a quiet build pass `QUIET=1`, for verbose pass `VERBOSE=1`.
 
@@ -126,7 +127,7 @@ Building your own image
 If you don't want to pull an untrusted image, you can build it yourself with:
 
 ```
-$ make rebuild-image@ppc64le@ubuntu@21.04
+$ make rebuild-image@ppc64le@ubuntu@21.10
 ```
 
 Note that the build mounts the source tree read-only, so nothing it does can

build/Makefile

@@ -58,11 +58,11 @@ DOCS += docs@${1}@${2}
 endef
 
 FEDORA_DISTROS := fedora@34 fedora@33 fedora@32 fedora@31
-UBUNTU_DISTROS := ubuntu@21.04 ubuntu@20.10 ubuntu@20.04 ubuntu@18.04 ubuntu@16.04
+UBUNTU_DISTROS := ubuntu@21.10 ubuntu@21.04 ubuntu@20.10 ubuntu@20.04 ubuntu@18.04 ubuntu@16.04
 KORG_DISTROS := korg@11.1.0 korg@10.3.0 korg@9.3.0 korg@8.1.0 korg@5.5.0
 ALL_DISTROS := ${UBUNTU_DISTROS} ${KORG_DISTROS} ${FEDORA_DISTROS}
-DOCS_DISTRO := docs@21.04
-X86_DISTRO := ubuntu@21.04
+DOCS_DISTRO := docs@21.10
+X86_DISTRO := ubuntu@21.10
 SUBARCHES := ppc64le ppc64
 
 $(eval $(call MAIN_TEMPLATE,ppc64le,${DOCS_DISTRO}))

build/ubuntu/packages.sh

@@ -6,14 +6,11 @@ machine=$(uname -m)
 
 . /etc/os-release
 
-if [[ "$VERSION_ID" == "21.04" ]]; then
+if [[ "$VERSION_ID" == "21.10" ]]; then
     if [[ "$machine" != "x86_64" ]]; then
         PACKAGES+=" crossbuild-essential-amd64"
     fi
-fi
 
-major="${VERSION_ID%%.*}"
-if [[ $major -ge 18 ]]; then
     PACKAGES+=" clang llvm"
 fi
 

lib/pexpect_utils.py

@@ -23,7 +23,7 @@ def __init__(self):
         self.bug_patterns = self.default_bug_patterns
 
     def spawn(self, *args, **kwargs):
-        logging.debug("Spawning '%s'" % args)
+        logging.info("Spawning '%s'" % args)
         self.child = pexpect.spawn(*args, timeout=60, encoding='utf-8', echo=False, **kwargs)
         if '--quiet' not in sys.argv:
             self.log_to(sys.stdout)
@@ -83,8 +83,8 @@ def pop_prompt(self):
         self.prompt_stack.pop()
         self.prompt = self.prompt_stack[-1]
 
-    def expect_prompt(self):
-        self.expect(self.prompt)
+    def expect_prompt(self, timeout=-1):
+        self.expect(self.prompt, timeout=timeout)
 
     def send_no_newline(self, data):
         self.child.send(data)
@@ -98,27 +98,34 @@ def cmd(self, cmd):
         self.expect_prompt()
 
 
-def standard_boot(p, login=False, user='root'):
-    p.push_prompt(p.DEFAULT_PROMPT)
+def standard_boot(p, login=False, user='root', password=None, timeout=-1, prompt=None):
+    if prompt is None:
+        prompt = p.DEFAULT_PROMPT
+
+    p.push_prompt(prompt)
 
     logging.info("Waiting for kernel to boot")
-    p.expect("Freeing unused kernel ")
+    p.expect("Freeing unused kernel ", timeout=timeout)
 
     if login:
         logging.info("Kernel came up, waiting for login ...")
-        p.expect("login:")
+        p.expect("login:", timeout=timeout)
         p.send(user)
+        if password is not None:
+            p.expect("Password:", timeout=timeout)
+            p.send(password)
     else:
         logging.info("Kernel came up, waiting for prompt ...")
 
-    p.expect_prompt()
+    p.expect_prompt(timeout=timeout)
     logging.info("Booted to shell prompt")
 
 
 def ping_test(p, ip='10.0.2.2', check=True):
-    p.send(f'ping -c 3 {ip}')
+    p.send(f'ping -W 10 -c 3 {ip}')
     if check:
-        p.expect('3 packets transmitted, 3 packets received')
+        # busybox ping prints "packets received", iputils-ping does not
+        p.expect('3 packets transmitted, 3( packets)? received')
     p.expect_prompt()
 
 

lib/qemu.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import logging
 from utils import get_env_var, get_endian
 from pexpect_utils import PexpectHelper
@@ -14,10 +15,22 @@ def get_qemu(name='qemu-system-ppc64'):
     return qemu
 
 
-def get_rootfs(fname):
-    path = get_env_var('ROOT_DISK_PATH', '')
-    val = os.path.join(path, fname)
+def get_root_disk_path():
+    path = get_env_var('ROOT_DISK_PATH', None)
+    if path is not None:
+        return path
 
+    base = os.path.dirname(sys.argv[0])
+    # Assumes we're called from scripts/boot/qemu-xxx
+    path = f'{base}/../../root-disks'
+    if os.path.isdir(path):
+        return path
+
+    return ''
+
+
+def get_root_disk(fname):
+    val = os.path.join(get_root_disk_path(), fname)
     logging.debug(f'Using rootfs {val}')
     return val
 
@@ -31,7 +44,7 @@ def get_qemu_version(emulator):
 
 
 def qemu_command(qemu='qemu-system-ppc64', machine='pseries,cap-htm=off', cpu=None,
-                 mem='1G', smp=1, vmlinux=None, rootfs=None,
+                 mem='1G', smp=1, vmlinux=None, initrd=None, drive=None,
                  cmdline='', accel='tcg', net='-nic user'):
 
     qemu_path = get_qemu(qemu)
@@ -40,16 +53,6 @@ def qemu_command(qemu='qemu-system-ppc64', machine='pseries,cap-htm=off', cpu=No
     if vmlinux is None:
         vmlinux = get_vmlinux()
 
-    if rootfs is None:
-        if qemu == 'qemu-system-ppc':
-            subarch = 'ppc'
-        elif get_endian(vmlinux) == 'little':
-            subarch = 'ppc64le'
-        else:
-            subarch = 'ppc64'
-
-        rootfs = f'{subarch}-rootfs.cpio.gz'
-
     l = [
         get_qemu(qemu),
         '-nographic',
@@ -59,10 +62,26 @@ def qemu_command(qemu='qemu-system-ppc64', machine='pseries,cap-htm=off', cpu=No
         '-m', mem,
         '-accel', accel,
         '-kernel', vmlinux,
-        '-initrd', get_rootfs(rootfs),
         net,
     ]
 
+    if initrd is None and drive is None:
+        if qemu == 'qemu-system-ppc':
+            subarch = 'ppc'
+        elif get_endian(vmlinux) == 'little':
+            subarch = 'ppc64le'
+        else:
+            subarch = 'ppc64'
+
+        initrd = f'{subarch}-rootfs.cpio.gz'
+
+    if initrd:
+        l.append('-initrd')
+        l.append(get_root_disk(initrd))
+
+    if drive:
+        l.append(drive)
+
     if cpu is not None:
         l.append('-cpu')
         l.append(cpu)

root-disks/.gitignore

@@ -1 +1,12 @@
 *.gz
+cloud-init-user-data.img
+ubuntu16.04-cloudimg-powerpc.qcow2
+ubuntu16.04-cloudimg-powerpc.qcow2.sum
+ubuntu16.04-cloudimg-ppc64el.qcow2
+ubuntu16.04-cloudimg-ppc64el.qcow2.sum
+ubuntu21.04-cloudimg-ppc64el.qcow2
+ubuntu21.04-cloudimg-ppc64el.qcow2.sum
+ubuntu21.10-cloudimg-ppc64el.qcow2
+ubuntu21.10-cloudimg-ppc64el.qcow2.sum
+fedora34-cloudimg-ppc64le.qcow2
+fedora34-cloudimg-ppc64le.qcow2.sum

root-disks/Makefile

@@ -1,18 +1,61 @@
-IMAGES += ppc64-rootfs.cpio.gz ppc64-novsx-rootfs.cpio.gz ppc64le-rootfs.cpio.gz ppc-rootfs.cpio.gz
+IMAGES += ppc64-rootfs.cpio.gz ppc64-novsx-rootfs.cpio.gz ppc64le-rootfs.cpio.gz
+IMAGES += ppc-rootfs.cpio.gz
+IMAGES += cloud-init-user-data.img
+IMAGES += ubuntu16.04-cloudimg-powerpc.qcow2
+IMAGES += ubuntu16.04-cloudimg-ppc64el.qcow2
+IMAGES += ubuntu21.04-cloudimg-ppc64el.qcow2
+IMAGES += ubuntu21.10-cloudimg-ppc64el.qcow2
+IMAGES += fedora34-cloudimg-ppc64le.qcow2
 
 all: $(IMAGES)
 
 ppc64-rootfs.cpio.gz: ppc64-novsx-rootfs.cpio.gz
 	ln -s $< $@
 
 ppc64-novsx-rootfs.cpio.gz:
-	wget -O $@ "https://github.com/groeck/linux-build-test/blob/4e693ba1cfc17ff4c551626855d2982c492be1c2/rootfs/ppc64/rootfs.cpio.gz?raw=true"
+	wget -O $@.tmp "https://github.com/groeck/linux-build-test/blob/4e693ba1cfc17ff4c551626855d2982c492be1c2/rootfs/ppc64/rootfs.cpio.gz?raw=true"
+	mv $@.tmp $@
 
 ppc64le-rootfs.cpio.gz:
-	wget -O $@ "https://github.com/groeck/linux-build-test/blob/4e693ba1cfc17ff4c551626855d2982c492be1c2/rootfs/ppc64/rootfs-el.cpio.gz?raw=true"
+	wget -O $@.tmp "https://github.com/groeck/linux-build-test/blob/4e693ba1cfc17ff4c551626855d2982c492be1c2/rootfs/ppc64/rootfs-el.cpio.gz?raw=true"
+	mv $@.tmp $@
 
 ppc-rootfs.cpio.gz:
-	wget -O $@ "https://github.com/groeck/linux-build-test/blob/65335f1a683166c7b83d61a3eeaf400ebcfa20dd/rootfs/ppc/rootfs.cpio.gz?raw=true"
+	wget -O $@.tmp "https://github.com/groeck/linux-build-test/blob/65335f1a683166c7b83d61a3eeaf400ebcfa20dd/rootfs/ppc/rootfs.cpio.gz?raw=true"
+	mv $@.tmp $@
+
+ubuntu16.04-cloudimg-powerpc.qcow2:
+	wget -O $@.tmp "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-powerpc-disk1.img"
+	mv $@.tmp $@
+	chmod a-w $@
+	sha256sum $@ > $@.sum
+
+ubuntu16.04-cloudimg-ppc64el.qcow2:
+	wget -O $@.tmp "https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-ppc64el-disk1.img"
+	mv $@.tmp $@
+	chmod a-w $@
+	sha256sum $@ > $@.sum
+
+ubuntu21.04-cloudimg-ppc64el.qcow2:
+	wget -O $@.tmp "https://cloud-images.ubuntu.com/hirsute/current/hirsute-server-cloudimg-ppc64el.img"
+	mv $@.tmp $@
+	chmod a-w $@
+	sha256sum $@ > $@.sum
+
+ubuntu21.10-cloudimg-ppc64el.qcow2:
+	wget -O $@.tmp "https://cloud-images.ubuntu.com/impish/current/impish-server-cloudimg-ppc64el.img"
+	mv $@.tmp $@
+	chmod a-w $@
+	sha256sum $@ > $@.sum
+
+fedora34-cloudimg-ppc64le.qcow2:
+	wget -O $@.tmp "https://download.fedoraproject.org/pub/fedora-secondary/releases/34/Cloud/ppc64le/images/Fedora-Cloud-Base-34-1.2.ppc64le.qcow2"
+	mv $@.tmp $@
+	chmod a-w $@
+	sha256sum $@ > $@.sum
+
+cloud-init-user-data.img: cloud-init-user-data.txt
+	cloud-localds $@ $<
 
 clean:
 	@echo "Use distclean to remove rootfs images"

root-disks/cloud-init-user-data.txt

@@ -0,0 +1,11 @@
+#cloud-config
+users:
+  - name: user
+    lock_passwd: false
+    plain_text_passwd: linuxppc
+  - name: root
+    lock_passwd: false
+    plain_text_passwd: linuxppc
+
+# don't regenerate ssh keys, it takes time for no benefit
+ssh_deletekeys: false

scripts/boot/qemu-44x

@@ -1,8 +1,5 @@
 #!/usr/bin/python3
 #
-# $ export ROOT_DISK_PATH=~/root-disks/
-# Expects ppc-rootfs.cpio.gz in ROOT_DISK_PATH
-#
 # $ cd ~/src/linux
 # $ make
 # $ ~/src/ci-scripts/scripts/boot/qemu-44x
@@ -14,6 +11,8 @@
 #
 # Optional:
 # export QEMU_SYSTEM_PPC=~/src/qemu/ppc-softmmu/qemu-system-ppc
+# export ROOT_DISK_PATH=~/some/directory
+# Expects ppc-rootfs.cpio.gz in ROOT_DISK_PATH
 
 import logging
 import os, sys
@@ -28,9 +27,6 @@ def main():
     setup_logging()
     setup_timeout(60)
 
-    if not check_env_vars(['ROOT_DISK_PATH']):
-        return 1
-
     expected_release = get_expected_release()
     if expected_release is None:
         return False