Extract: add support for rootfs and debootstrap

Author: solcloud

Makefile

@@ -4,7 +4,7 @@ all:
 	@echo "make build|linux|busybox|initrd|target|ssh|sync|test|check|clean|download|up|extract|vbox|cmd|gui"
 
 extract:
-	./scripts/extract_from_vm.sh && ./scripts/extract.sh
+	./scripts/extract_signpost.sh
 
 vbox:
 	./scripts/vbox_disk_generate.sh

README.md

@@ -40,7 +40,7 @@ make build
 make gui
 ```
 
-For extracting binaries from other distribution just look at [distro_extractor](distro_extractor/) folder. We virtually provide extract recipes for every linux distro ever made 😉 For example if you prefer _debian_ binaries over _arch_ just use something like `DISTRO=devuan DISTRO_ISO=/path/to/devuan_chimaera_4.0.0_amd64_minimal-live.iso make extract`
+For extracting binaries from other distribution just read [supported distributions](distro_extractor/README.md). We virtually provide extract recipes for every linux distro ever made 😉 For example if you prefer _debian_ binaries over _arch_ just use something like `DISTRO=devuan DISTRO_ISO=/path/to/devuan_chimaera_4.0.0_amd64_minimal-live.iso make extract`. For more extract methods run `make extract`.
 
 ## User's config
 

distro_extractor/README.md

@@ -0,0 +1,37 @@
+## Extracting packages from other distributions
+
+NiceOS do not provide package manager. Instead, it uses other distributions package managers.
+
+Currently, we support **3** extract methods:
+- Virtual machine (**vm**)
+  - our recommended method for extracting
+  - you need virtual machine provider (_QEMU_ or _VirtualBox_) and distribution install _ISO_
+- Root file system (**rfs**)
+  - installing by extracting distribution _rootfs_ into temp folder and perform _chroot_ install there 
+- Debootstrap (**deb**)
+  - for debian based distributions using _debootstrap_ binary
+
+## Supported distributions
+
+NiceOS support out of the box these distributions (alphabetical order):
+
+---
+**NOTE:**
+Always check firstly for latest version for your architecture and desired flavour on distribution homepage or use your specific version instead of example one.
+Also, many distributions provide mirrors at different geographical places and URLs - try to use closest and fastest mirror for you.
+---
+
+
+- Artix (https://artixlinux.org/)
+  - vm `DISTRO=artix DISTRO_ISO=/path/to/artix-base-openrc-20220123-x86_64.iso make extract`
+- Debian (https://www.debian.org/)
+  - deb: `DEBOOTSTRAP_SUITE=stable DEBOOTSTRAP_MIRROR='https://deb.debian.org/debian/' make extract`
+- Devuan (https://www.devuan.org/)
+  - vm: `DISTRO=devuan DISTRO_ISO=/path/to/devuan_chimaera_4.0.0_amd64_minimal-live.iso make extract`
+  - deb: `DEBOOTSTRAP_SUITE=stable DEBOOTSTRAP_MIRROR='https://packages.devuan.org/merged' make extract`
+- Ubuntu (https://ubuntu.com/)
+  - rfs: `DISTRO=debian-based DISTRO_ROOTFS=/path/to/ubuntu-base-21.10-base-amd64.tar.gz make extract`
+  - deb: `DEBOOTSTRAP_SUITE=jammy DEBOOTSTRAP_SCRIPT=stable DEBOOTSTRAP_MIRROR='http://archive.ubuntu.com/ubuntu' make extract`
+- Void (https://voidlinux.org/)
+  - vm: `DISTRO=void DISTRO_ISO=/path/to/void-live-x86_64-20210930.iso make extract`
+  - rfs: `DISTRO=void DISTRO_ROOTFS=/path/to/void-x86_64-ROOTFS-20210930.tar.xz make extract`

distro_extractor/artix/inc.sh

@@ -17,3 +17,8 @@ function boot_info_qemu() {
     echo "Run '/bin/sshd'"
     echo "For future password prompt write $VM_PASS"
 }
+
+function distro_install_command() {
+    packages=$(cat "$NICE_PRESET_PATH/packages.${PM}.txt" | xargs)
+    echo "pacman -Sy --noconfirm $packages"
+}

distro_extractor/debian-based/inc.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+PM=deb
+
+function distro_install_command() {
+    packages=$(cat "$NICE_PRESET_PATH/packages.${PM}.txt" | xargs)
+    echo "apt-get update && apt-get install --no-install-recommends --assume-yes $packages"
+}

distro_extractor/devuan/inc.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-PM=deb
+source distro_extractor/debian-based/inc.sh || exit 1
+
 VM_USER=devuan
 VM_PASS=$VM_USER
 

distro_extractor/void/inc.sh

@@ -9,3 +9,8 @@ function boot_info() {
     echo "Start boot by pressing enter or wait 10 sec"
     echo "After boot finished"
 }
+
+function distro_install_command() {
+    packages=$(cat "$NICE_PRESET_PATH/packages.${PM}.txt" | xargs)
+    echo "xbps-install --yes -S $packages"
+}

scripts/extract_from_chroot.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+source ./.config.sh || exit 1
+
+source "$BASE/distro_extractor/$DISTRO/inc.sh" || dd "File '$BASE/distro_extractor/$DISTRO/inc.sh' cannot be sourced"
+command=$(distro_install_command)
+
+notify "We need sudo for bind mounting, chroot and cleanup"
+sudo mount --bind /sys "$VM_MOUNT_ROOT/sys"
+sudo mount --bind /dev "$VM_MOUNT_ROOT/dev"
+sudo mount --bind /proc "$VM_MOUNT_ROOT/proc"
+sudo cp /etc/resolv.conf "$VM_MOUNT_ROOT/etc/"
+
+sudo chroot "$VM_MOUNT_ROOT" /bin/sh -c "'$command'"
+
+sudo umount "$VM_MOUNT_ROOT/proc"
+sudo umount "$VM_MOUNT_ROOT/dev"
+sudo umount "$VM_MOUNT_ROOT/sys"

scripts/extract_from_vm.sh

@@ -2,20 +2,6 @@
 
 source ./.config.sh || exit 1
 
-HYPERVISOR=${HYPERVISOR:-'qemu'}
-if [ -z "$DISTRO" ]; then
-    echo "You need to specify extracting distribution from $BASE/distro_extractor, use one of"
-    ls "$BASE/distro_extractor"
-    dd "use \`export DISTRO=artix\` for example"
-fi
-if [ -z "$DISTRO_ISO" ]; then
-    echo "You need to specify distribution install iso path"
-    echo "use \`export DISTRO_ISO=/data/dwn/artix-base-openrc-20220123-x86_64.iso\` for example"
-    echo " or "
-    dd "use \`export DISTRO_ISO=/data/dwn/devuan_chimaera_4.0.0_amd64_minimal-live.iso\` for example"
-fi
-
-
 function boot_info_qemu() {
     echo "For future password prompt write $VM_PASS"
 }

scripts/extract_signpost.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+source ./.config.sh || exit 1
+
+function requires_distro_variable() {
+    if [ -z "$DISTRO" ]; then
+        echo "You need to specify extracting distribution from $BASE/distro_extractor, use one of"
+        ls "$BASE/distro_extractor"
+        dd "use \`export DISTRO=artix\` for example"
+    fi
+}
+
+
+if [ -n "$DISTRO_ISO" ]; then
+    requires_distro_variable
+    which VBoxManage &> /dev/null && HYPERVISOR_CANDIDATE=virtualbox || true
+    which qemu-system-x86_64 &> /dev/null && HYPERVISOR_CANDIDATE=qemu || true
+
+    export HYPERVISOR=${HYPERVISOR:-"$HYPERVISOR_CANDIDATE"}
+    ./scripts/extract_from_vm.sh
+    ./scripts/extract.sh
+
+elif [ -n "$DISTRO_ROOTFS" ]; then
+    requires_distro_variable
+    export NICE_EXTRACT_DISTRO_HDD_IMAGE_PATH=/tmp/nice-not-exists.noimg
+    export VM_MOUNT_ROOT="$STORAGE/temp/extract/rootfs_$(date +%s)" && mkdir -p "$VM_MOUNT_ROOT"
+    tar --checkpoint=100 -xf "$DISTRO_ROOTFS" -C "$VM_MOUNT_ROOT"
+    ./scripts/extract_from_chroot.sh
+    ./scripts/extract.sh
+    sudo rm -rf "$VM_MOUNT_ROOT"
+
+elif [ -n "$DEBOOTSTRAP_SUITE" ]; then
+    which debootstrap > /dev/null || dd "Debootstrap binary not found"
+
+    export VM_MOUNT_ROOT="$STORAGE/temp/extract/debootstrap_$(date +%s)" && mkdir -p "$VM_MOUNT_ROOT"
+    export NICE_EXTRACT_DISTRO_HDD_IMAGE_PATH=/tmp/nice-not-exists.noimg
+    notify "We need sudo for debootstrap and cleanup"
+    sudo debootstrap --variant=minbase --merged-usr --arch="${ARCH:-amd64}" --include="$(cat "$NICE_PRESET_PATH/packages.deb.txt" | xargs | sed 's/ /,/g')" "$DEBOOTSTRAP_SUITE" "$VM_MOUNT_ROOT" "$DEBOOTSTRAP_MIRROR" "$DEBOOTSTRAP_SCRIPT"
+    ./scripts/extract.sh
+    sudo rm -rf "$VM_MOUNT_ROOT"
+
+else
+    echo "You need to provide extract method. Use one of:"
+    echo "'DISTRO_ISO' OR 'DISTRO_ROOTFS' OR 'DEBOOTSTRAP_SUITE'"
+    dd "For example use \`export DISTRO_ISO=/data/dwn/artix-base-openrc-20220123-x86_64.iso\`"
+fi