Add option to require explicit gpg key verification for image builds (#15462)

Signed-off-by: Daniel McIlvaney <damcilva@microsoft.com>

Author: dmcilvaney

toolkit/Makefile

@@ -164,8 +164,13 @@ else
 VALIDATE_TOOLCHAIN_GPG ?= y
 endif
 endif
+##help:var:VALIDATE_IMAGE_GPG:{y,n}=Enable RPM GPG signature verification during package fetching and image builds. When enabled, all packages must be signed - this validates that packages have completed the signing process. Default is 'n' for local development with unsigned packages. Production builds use a multi-step workflow (build packages -> sign packages -> build images) and should set 'y' for the final image build step to enforce that all packages are signed. Keys used for validation can be modified with the IMAGE_GPG_VALIDATION_KEYS variable.
+VALIDATE_IMAGE_GPG ?= n
 
-TOOLCHAIN_GPG_VALIDATION_KEYS ?= $(wildcard $(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY) $(wildcard $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY)
+# Default GPG keys for package GPG validation, used with VALIDATE_TOOLCHAIN_GPG and VALIDATE_IMAGE_GPG
+default_gpg_keys := $(wildcard $(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY) $(wildcard $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY)
+TOOLCHAIN_GPG_VALIDATION_KEYS ?= $(default_gpg_keys)
+IMAGE_GPG_VALIDATION_KEYS ?= $(default_gpg_keys)
 
 ######## COMMON MAKEFILE UTILITIES ########
 

toolkit/docs/building/building.md

@@ -865,6 +865,10 @@ Authentication mode for downloading source files for SRPM packing. Valid options
 | INCREMENTAL_TOOLCHAIN            | n                                                                                                      | Only build toolchain RPM packages if they are not already present
 | RUN_CHECK                        | n                                                                                                      | Run the %check sections when compiling packages
 | ALLOW_TOOLCHAIN_REBUILDS         | n                                                                                                      | Do not treat rebuilds of toolchain packages during regular package build phase as errors.
+| VALIDATE_TOOLCHAIN_GPG           | (auto - based on toolchain build mode)                                                                 | Enable RPM GPG signature verification for toolchain packages. Automatically set to `y` when downloading pre-built toolchain packages (`REBUILD_TOOLCHAIN=n`), and `n` when rebuilding locally or using `DAILY_BUILD_ID`. Packages are validated against keys specified in `TOOLCHAIN_GPG_VALIDATION_KEYS`.
+| TOOLCHAIN_GPG_VALIDATION_KEYS    | `$(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY` | Space separated list of GPG key files used to validate RPM signatures when `VALIDATE_TOOLCHAIN_GPG=y`.
+| VALIDATE_IMAGE_GPG               | n                                                                                                      | Enable RPM GPG signature verification during image builds. When set to `y`, all packages fetched for image generation must have valid GPG signatures. Packages are validated against keys specified in `IMAGE_GPG_VALIDATION_KEYS`. Production builds should enable this to ensure all packages have completed the signing process.
+| IMAGE_GPG_VALIDATION_KEYS        | `$(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY` | Space separated list of GPG key files used to validate RPM signatures when `VALIDATE_IMAGE_GPG=y`.
 |  PACKAGE_BUILD_RETRIES           | 1                                                                                                      | Number of build retries for each package
 | CHECK_BUILD_RETRIES              | 1                                                                                                      | Minimum number of check section retries for each package if RUN_CHECK=y and tests fail.
 | MAX_CASCADING_REBUILDS           |                                                                                                        | When a package rebuilds, how many additional layers of dependent packages will be forced to rebuild (leave unset for unbounded, i.e., all downstream packages will rebuild)

toolkit/docs/security/intro.md

@@ -4,3 +4,5 @@ Below topics are dedicated to security-related details of the operating system.
 
 ## 1. [Security features](security-features.md)
 ## 2. [SSL CA certificates management](ca-certificates.md)
+## 3. [Verifying ISO images](iso-image-verification.md)
+## 4. [Production build recommendations](production-builds.md)

toolkit/docs/security/production-builds.md

@@ -0,0 +1,30 @@
+# Production Build Recommendations
+
+When building images or ISOs for production deployment, enable explicit GPG signature verification to ensure all packages have completed the signing process:
+
+```bash
+sudo make image VALIDATE_IMAGE_GPG=y CONFIG_FILE=<your-config>
+```
+
+This validates that all RPM packages fetched during image generation have valid GPG signatures from the expected signing keys.
+
+## Build Workflow
+
+A typical production workflow separates package building from image generation:
+
+1. **Build packages** - Compile packages from source
+2. **Sign packages** - Sign built packages with your GPG key
+3. **Build images** - Generate images with `VALIDATE_IMAGE_GPG=y` to enforce all packages are signed
+
+This separation ensures unsigned or improperly signed packages cannot be included in final images.
+
+## Related Variables
+
+| Variable | Description |
+|:---------|:------------|
+| `VALIDATE_IMAGE_GPG` | Set to `y` to require valid GPG signatures on all image packages |
+| `IMAGE_GPG_VALIDATION_KEYS` | GPG key files for signature validation |
+| `VALIDATE_TOOLCHAIN_GPG` | Automatically enabled when downloading pre-built toolchain |
+| `TOOLCHAIN_GPG_VALIDATION_KEYS` | GPG key files for toolchain validation |
+
+See [build variables](../building/building.md#all-build-variables) for full details.

toolkit/scripts/imggen.mk

@@ -140,7 +140,12 @@ ifneq ($(REPO_SNAPSHOT_TIME),)
 imagepkgfetcher_extra_flags += --repo-snapshot-time=$(REPO_SNAPSHOT_TIME)
 endif
 
-$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(toolchain_rpms) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms) $(depend_REPO_SNAPSHOT_TIME) $(STATUS_FLAGS_DIR)/imagegen_cleanup.flag
+ifeq ($(VALIDATE_IMAGE_GPG),y)
+imagepkgfetcher_extra_flags += --enable-gpg-check
+imagepkgfetcher_extra_flags += $(foreach key,$(IMAGE_GPG_VALIDATION_KEYS),--gpg-key=$(key))
+endif
+
+$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(toolchain_rpms) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms) $(depend_REPO_SNAPSHOT_TIME) $(depend_VALIDATE_IMAGE_GPG) $(depend_IMAGE_GPG_VALIDATION_KEYS) $(IMAGE_GPG_VALIDATION_KEYS) $(STATUS_FLAGS_DIR)/imagegen_cleanup.flag
 	$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
 	$(go-imagepkgfetcher) \
 		--input=$(CONFIG_FILE) \

toolkit/scripts/toolchain.mk

@@ -309,7 +309,7 @@ $(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_local_tem
 
 # No archive was selected, so download from online package server instead. All packages must be available for this step to succeed.
 else
-$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_auto_cleanup.flag $(depend_REBUILD_TOOLCHAIN) $(go-downloader) $(SCRIPTS_DIR)/toolchain/download_toolchain_rpm.sh $(TOOLCHAIN_GPG_VALIDATION_KEYS)
+$(toolchain_rpms): $(TOOLCHAIN_MANIFEST) $(STATUS_FLAGS_DIR)/toolchain_auto_cleanup.flag $(depend_REBUILD_TOOLCHAIN) $(go-downloader) $(SCRIPTS_DIR)/toolchain/download_toolchain_rpm.sh $(depend_TOOLCHAIN_GPG_VALIDATION_KEYS) $(TOOLCHAIN_GPG_VALIDATION_KEYS)
 	@log_file="$(toolchain_downloads_logs_dir)/$(notdir $@).log" && \
 	rm -f "$$log_file" && \
 	$(SCRIPTS_DIR)/toolchain/download_toolchain_rpm.sh \

toolkit/scripts/utils.mk

@@ -66,10 +66,11 @@ endef
 ######## VARIABLE DEPENDENCY TRACKING ########
 
 # List of variables to watch for changes.
-watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR MAX_CASCADING_REBUILDS RUN_CHECK TEST_RUN_LIST TEST_RERUN_LIST TEST_IGNORE_LIST EXTRA_BUILD_LAYERS LICENSE_CHECK_MODE VALIDATE_TOOLCHAIN_GPG REPO_SNAPSHOT_TIME PACKAGE_CACHE_SUMMARY
+watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR MAX_CASCADING_REBUILDS RUN_CHECK TEST_RUN_LIST TEST_RERUN_LIST TEST_IGNORE_LIST EXTRA_BUILD_LAYERS LICENSE_CHECK_MODE VALIDATE_TOOLCHAIN_GPG TOOLCHAIN_GPG_VALIDATION_KEYS VALIDATE_IMAGE_GPG IMAGE_GPG_VALIDATION_KEYS REPO_SNAPSHOT_TIME PACKAGE_CACHE_SUMMARY
 # Current list: $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_REPO_LIST) $(depend_CONFIG_FILE) $(depend_STOP_ON_PKG_FAIL)
 #					$(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST) $(depend_SPECS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_MAX_CASCADING_REBUILDS) $(depend_RUN_CHECK) $(depend_TEST_RUN_LIST)
-#					$(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(depend_LICENSE_CHECK_MODE) $(depend_VALIDATE_TOOLCHAIN_GPG) $(depend_REPO_SNAPSHOT_TIME) $(depend_PACKAGE_CACHE_SUMMARY)
+#					$(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(depend_LICENSE_CHECK_MODE) $(depend_VALIDATE_TOOLCHAIN_GPG) $(depend_TOOLCHAIN_GPG_VALIDATION_KEYS) $(depend_VALIDATE_IMAGE_GPG)
+#					$(depend_IMAGE_GPG_VALIDATION_KEYS) $(depend_REPO_SNAPSHOT_TIME) $(depend_PACKAGE_CACHE_SUMMARY)
 
 .PHONY: variable_depends_on_phony clean-variable_depends_on_phony setfacl_always_run_phony
 clean: clean-variable_depends_on_phony

toolkit/tools/imagegen/installutils/installutils.go

@@ -766,7 +766,9 @@ func TdnfInstallWithProgress(packageName, installRoot string, currentPackagesIns
 		return
 	}
 
-	// TDNF 3.x uses repositories from installchroot instead of host. Passing setopt for repo files directory to use local repo for installroot installation
+	// TDNF 3.x uses repositories from installchroot instead of host. Passing setopt for repo files directory to use local repo for installroot installation.
+	// Note: --nogpgcheck is used here because GPG signature validation is performed earlier during package fetching (imagepkgfetcher)
+	// when VALIDATE_IMAGE_GPG=y is set. Packages in the local repo have already been verified.
 	err = shell.NewExecBuilder("tdnf", "-v", "install", packageName, "--installroot", installRoot, "--nogpgcheck",
 		"--assumeyes", "--setopt", "reposdir=/etc/yum.repos.d/", releaseverCliArg).
 		StdoutCallback(onStdout).
@@ -830,7 +832,9 @@ func calculateTotalPackages(packages []string, installRoot string) (installedPac
 			stderr string
 		)
 
-		// Issue an install request but stop right before actually performing the install (assumeno)
+		// Issue an install request but stop right before actually performing the install (assumeno).
+		// Note: --nogpgcheck is safe here because this is a dry-run (--assumeno) and packages are validated
+		// during fetching when VALIDATE_IMAGE_GPG=y is set.
 		stdout, stderr, err = shell.Execute("tdnf", "install", releaseverCliArg, "--assumeno", "--nogpgcheck", pkg, "--installroot", installRoot)
 		if err != nil {
 			// tdnf aborts the process when it detects an install with --assumeno.

toolkit/tools/imagepkgfetcher/imagepkgfetcher.go

@@ -16,6 +16,7 @@ import (
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/packagerepo/repoutils"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/pkggraph"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/pkgjson"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/rpm"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/timestamp"
 	"github.com/microsoft/azurelinux/toolkit/tools/pkg/profile"
 
@@ -49,6 +50,9 @@ var (
 	inputSummaryFile  = app.Flag("input-summary-file", "Path to a file with the summary of packages cloned to be restored").String()
 	outputSummaryFile = app.Flag("output-summary-file", "Path to save the summary of packages cloned").String()
 
+	enableGpgCheck = app.Flag("enable-gpg-check", "Enable RPM GPG signature verification for all repositories during package fetching.").Bool()
+	gpgKeyPaths    = app.Flag("gpg-key", "Path to a GPG key file for signature validation. May be specified multiple times. Required if enable-gpg-check is set.").ExistingFiles()
+
 	logFlags      = exe.SetupLogFlags(app)
 	profFlags     = exe.SetupProfileFlags(app)
 	timestampFile = app.Flag("timestamp-file", "File that stores timestamps for this program.").String()
@@ -73,6 +77,10 @@ func main() {
 		logger.Log.Fatal("input-graph must be provided if external-only is set.")
 	}
 
+	if *enableGpgCheck && len(*gpgKeyPaths) == 0 {
+		logger.Log.Fatal("--enable-gpg-check requires at least one --gpg-key path")
+	}
+
 	timestamp.StartEvent("initialize and configure cloner", nil)
 
 	cloner, err := rpmrepocloner.ConstructCloner(*outDir, *tmpDir, *workertar, *existingRpmDir, *existingToolchainRpmDir, *tlsClientCert, *tlsClientKey, *repoFiles, *repoSnapshotTime)
@@ -110,6 +118,14 @@ func main() {
 		logger.Log.Panicf("Failed to clone RPM repo. Error: %s", err)
 	}
 
+	// Validate GPG signatures of downloaded packages if enabled
+	if *enableGpgCheck {
+		err = rpm.ValidateDirectoryRPMSignatures(cloner.CloneDirectory(), *gpgKeyPaths)
+		if err != nil {
+			logger.Log.Panicf("Failed to validate RPM signatures. Error: %s", err)
+		}
+	}
+
 	timestamp.StartEvent("finalize cloned packages", nil)
 
 	err = cloner.ConvertDownloadedPackagesIntoRepo()

toolkit/tools/internal/rpm/rpm.go

@@ -5,6 +5,8 @@ package rpm
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -501,6 +503,91 @@ func InstallRPM(rpmFile string) (err error) {
 	return
 }
 
+const rpmKeysProgram = "rpmkeys"
+
+// importGPGKeysToRPMDb imports GPG keys into an RPM database for signature verification.
+// - rpmDbRoot: path to a directory to use as the RPM database root (will be created if it doesn't exist)
+// - gpgKeyPaths: paths to GPG key files to import into the RPM database
+// This should be called once before validating multiple RPMs with checkRPMSignature.
+func importGPGKeysToRPMDb(rpmDbRoot string, gpgKeyPaths []string) (err error) {
+	if _, err := exec.LookPath(rpmKeysProgram); err != nil {
+		return fmt.Errorf("%s command not found - explicit GPG signature enforcement requires this tool:\n%w", rpmKeysProgram, err)
+	}
+	for _, keyPath := range gpgKeyPaths {
+		_, stderr, importErr := shell.Execute(rpmKeysProgram, "--root", rpmDbRoot, "--import", keyPath)
+		if importErr != nil {
+			return fmt.Errorf("failed to import GPG key (%s) into RPM database: %v:\n%w", keyPath, stderr, importErr)
+		}
+	}
+	return nil
+}
+
+// checkRPMSignature validates the GPG signature of an RPM file.
+// - rpmFile: path to the RPM file to validate
+// - rpmDbRoot: path to a directory used as the RPM database root (must have GPG keys already imported via importGPGKeysToRpmDb)
+// Returns an error if the RPM signature is missing or invalid.
+func checkRPMSignature(rpmFile string, rpmDbRoot string) (err error) {
+	_, stderr, err := shell.Execute(rpmKeysProgram, "--root", rpmDbRoot, "--checksig", rpmFile, "-D", "%_pkgverify_level signature")
+	if err != nil {
+		return fmt.Errorf("RPM signature validation failed for (%s): %v\n%w", rpmFile, stderr, err)
+	}
+	return nil
+}
+
+// ValidateDirectoryRPMSignatures validates the GPG signatures of all RPM files in a directory.
+// It creates an isolated RPM database, imports the provided GPG keys, and validates each RPM.
+// Returns an error if any RPM has a missing or invalid signature.
+func ValidateDirectoryRPMSignatures(rpmDir string, gpgKeyPaths []string) (err error) {
+	logger.Log.Info("Validating GPG signatures of downloaded packages")
+
+	// Create a temporary directory for the isolated RPM database
+	rpmDbRoot, err := os.MkdirTemp("", "rpm-gpg-check-*")
+	if err != nil {
+		return fmt.Errorf("failed to create temporary directory for RPM database:\n%w", err)
+	}
+	defer os.RemoveAll(rpmDbRoot)
+
+	// Import GPG keys once before validating all RPMs
+	err = importGPGKeysToRPMDb(rpmDbRoot, gpgKeyPaths)
+	if err != nil {
+		return err
+	}
+
+	// Find all RPM files in the directory (recursively)
+	var rpmFiles []string
+	err = filepath.WalkDir(rpmDir, func(path string, d os.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if !d.IsDir() && filepath.Ext(path) == ".rpm" {
+			rpmFiles = append(rpmFiles, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to find RPM files in (%s):\n%w", rpmDir, err)
+	}
+
+	if len(rpmFiles) == 0 {
+		logger.Log.Debug("No RPM files found to validate")
+		return nil
+	}
+
+	logger.Log.Infof("Validating %d RPM files", len(rpmFiles))
+
+	// Validate each RPM
+	for _, rpmFile := range rpmFiles {
+		logger.Log.Debugf("Validating signature of: %s", filepath.Base(rpmFile))
+		err = checkRPMSignature(rpmFile, rpmDbRoot)
+		if err != nil {
+			return fmt.Errorf("GPG signature validation failed:\n%w", err)
+		}
+	}
+
+	logger.Log.Info("All downloaded RPMs have valid GPG signatures")
+	return nil
+}
+
 // QueryRPMProvides returns what an RPM file provides.
 // This includes any provides made by a generator and files provided by the rpm.
 func QueryRPMProvides(rpmFile string) (provides []string, err error) {