diff --git a/Cargo.lock b/Cargo.lock index a08a2de10..dd1874510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1703,9 +1703,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", diff --git a/rust/boil/README.md b/rust/boil/README.md index 366b49646..f9851cedb 100644 --- a/rust/boil/README.md +++ b/rust/boil/README.md @@ -31,3 +31,28 @@ boil image list # Display a list of versions of the image located in the 'airflow' folder boil image list airflow ``` + +## Advanced Building Options + +### Use Remote Cache + +> [!NOTE] +> The default builder (with the `docker` driver) doesn't support the registry cache storage backend. You must create +> a new builder using the `docker-container` driver and either set this new builder as the default or pass +> `-- --builder ` to use it: +> +> ```shell +> docker builder create --name container --driver=docker-container +> boil build airflow --cache-registry oci.example.org -- --builder container +> ``` + +boil offers to option to automatically pull from and push to a remote cache. This feature can be +enabled by using the `--cache-registry` (and the optional `--cache-namespace`) argument: + +```shell +# This will use `oci.example.org/-cache/airflow` to store and retrieve cached layers +boil build airflow --cache-registry oci.example.org + +# This will use `oci.example.org/foo/airflow` to store and retrieve cached layers +boil build airflow --cache-registry oci.example.org --cache-namespace foo +``` diff --git a/rust/boil/src/cli/build.rs b/rust/boil/src/cli/build.rs index 3409c92a0..253881801 100644 --- a/rust/boil/src/cli/build.rs +++ b/rust/boil/src/cli/build.rs @@ -118,6 +118,12 @@ pub struct BuildArguments { #[deprecated(since = "0.1.7", note = "Use -- --load instead")] pub load: bool, + #[arg(long, help_heading = "Build Options", group = "cache")] + pub cache_registry: Option, + + #[arg(long, help_heading = "Build Options", requires = "cache")] + pub cache_namespace: Option, + /// Dry run. This does not build the image(s) but instead prints out the bakefile. #[arg(short, long, alias = "dry")] pub dry_run: bool, diff --git a/rust/boil/src/core/bakefile.rs b/rust/boil/src/core/bakefile.rs index 649af56b9..9a1f21c13 100644 --- a/rust/boil/src/core/bakefile.rs +++ b/rust/boil/src/core/bakefile.rs @@ -1,6 +1,6 @@ use std::{ collections::{BTreeMap, btree_map::Entry}, - fmt::Debug, + fmt::{Debug, Display}, ops::{Deref, DerefMut}, path::PathBuf, }; @@ -340,26 +340,45 @@ impl Bakefile { for (image_name, (image_config, is_entry)) in targets.into_iter() { for (image_version, image_options) in image_config.versions { - let image_repository_uri = utils::format_image_repository_uri( + let ( + image_repository_uri, + image_index_manifest_tag, + image_manifest_tag, + image_manifest_uri, + ) = utils::format_image_tag_parts( image_registry, &cli_args.registry_namespace, &image_name, - ); - - let image_index_manifest_tag = utils::format_image_index_manifest_tag( &image_version, &metadata.vendor_tag_prefix, &cli_args.image_version, - ); - - let image_manifest_tag = utils::format_image_manifest_tag( - &image_index_manifest_tag, cli_args.target_platform.architecture(), cli_args.strip_architecture, ); - let image_manifest_uri = - utils::format_image_manifest_uri(&image_repository_uri, &image_manifest_tag); + let cache_image_manifest_uri = + cli_args.cache_registry.as_ref().map(|cache_registry| { + let uri = utils::format_image_cache_repository_uri( + cache_registry, + cli_args.cache_namespace.as_deref(), + &cli_args.registry_namespace, + &image_name, + ); + + let cache_image_index_manifest_tag = utils::format_image_index_manifest_tag( + &image_version, + &metadata.vendor_tag_prefix, + &cli_args.image_version, + ); + + let cache_image_manifest_tag = utils::format_image_manifest_tag( + &cache_image_index_manifest_tag, + cli_args.target_platform.architecture(), + cli_args.strip_architecture, + ); + + utils::format_image_manifest_uri(&uri, &cache_image_manifest_tag) + }); // TODO (@Techassi): Clean this up // TODO (@Techassi): Move the arg formatting into functions @@ -449,6 +468,23 @@ impl Bakefile { &cli_args.image_version, ); + let cache_to = + cache_image_manifest_uri + .clone() + .map_or_else(Vec::new, |reference| { + vec![CacheStorageBackend::Registry { + reference, + mode: Some(CacheMode::Max), + }] + }); + + let cache_from = cache_image_manifest_uri.map_or_else(Vec::new, |reference| { + vec![CacheStorageBackend::Registry { + reference, + mode: None, + }] + }); + let target = BakefileTarget { tags: vec![image_manifest_uri], arguments: build_arguments, @@ -459,6 +495,8 @@ impl Bakefile { inherits: vec![COMMON_TARGET_NAME.to_owned()], annotations, contexts, + cache_from, + cache_to, ..Default::default() }; @@ -529,6 +567,7 @@ impl Bakefile { // TODO (@Techassi): Figure out of we can use borrowed data in here. This would avoid a whole bunch // of cloning. #[derive(Debug, Default, Serialize)] +#[serde(rename_all = "kebab-case")] pub struct BakefileTarget { /// Defines build arguments for the target. #[serde( @@ -577,6 +616,12 @@ pub struct BakefileTarget { /// Image names and tags to use for the build target. #[serde(skip_serializing_if = "Vec::is_empty")] pub tags: Vec, + + #[serde(skip_serializing_if = "Vec::is_empty")] + pub cache_from: Vec, + + #[serde(skip_serializing_if = "Vec::is_empty")] + pub cache_to: Vec, } impl BakefileTarget { @@ -668,3 +713,54 @@ impl BakefileTarget { pub struct BakefileGroup { targets: Vec, } + +#[derive(Clone, Debug)] +pub enum CacheStorageBackend { + Registry { + /// Full name of the cache image to import. + reference: String, + + /// Specifies which layers to cache + mode: Option, + }, +} + +impl Display for CacheStorageBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CacheStorageBackend::Registry { reference, mode } => { + f.write_str("type=registry")?; + f.write_fmt(format_args!(",ref={reference}"))?; + + if let Some(mode) = mode { + f.write_fmt(format_args!(",mode={mode}"))?; + } + + Ok(()) + } + } + } +} + +impl Serialize for CacheStorageBackend { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +#[derive(Clone, Copy, Debug)] +pub enum CacheMode { + // We currently only support max, because we want to cache every layer + Max, +} + +impl Display for CacheMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CacheMode::Max => write!(f, "max"), + } + } +} diff --git a/rust/boil/src/utils.rs b/rust/boil/src/utils.rs index 895eec6a8..b43f71050 100644 --- a/rust/boil/src/utils.rs +++ b/rust/boil/src/utils.rs @@ -11,6 +11,24 @@ pub fn format_image_repository_uri( format!("{image_registry}/{registry_namespace}/{image_name}") } +pub fn format_image_cache_repository_uri( + image_cache_registry: &HostPort, + cache_registry_namespace: Option<&str>, + registry_namespace: &str, + image_name: &str, +) -> String { + // We don't use .map here because we are unable to borrow the formatted string long enough + if let Some(cache_registry_namespace) = cache_registry_namespace { + format_image_repository_uri(image_cache_registry, cache_registry_namespace, image_name) + } else { + format_image_repository_uri( + image_cache_registry, + &format!("{}-cache", registry_namespace), + image_name, + ) + } +} + /// Formats and returns the image manifest URI, eg. `oci.stackable.tech/sdp/opa:1.4.2-stackable25.7.0-amd64`. pub fn format_image_manifest_uri(image_repository_uri: &str, image_manifest_tag: &str) -> String { format!("{image_repository_uri}:{image_manifest_tag}") @@ -41,6 +59,35 @@ pub fn format_image_manifest_tag( } } +// TODO (@Techassi): Can we design this better? Maybe add a new struct/type which implements a bunch +// of associated functions. +#[allow(clippy::too_many_arguments)] +pub fn format_image_tag_parts( + image_registry: &HostPort, + registry_namespace: &str, + image_name: &str, + image_version: &str, + vendor_tag_prefix: &str, + vendor_image_version: &str, + architecture: &Architecture, + strip_architecture: bool, +) -> (String, String, String, String) { + let image_repository_uri = + format_image_repository_uri(image_registry, registry_namespace, image_name); + let image_index_manifest_tag = + format_image_index_manifest_tag(image_version, vendor_tag_prefix, vendor_image_version); + let image_manifest_tag = + format_image_manifest_tag(&image_index_manifest_tag, architecture, strip_architecture); + let image_manifest_uri = format_image_manifest_uri(&image_repository_uri, &image_manifest_tag); + + ( + image_repository_uri, + image_index_manifest_tag, + image_manifest_tag, + image_manifest_uri, + ) +} + /// Formats and returns the registry-specific env var name, eg. `BOIL_REGISTRY_TOKEN_OCI_STACKABLE_TECH`. pub fn format_registry_token_env_var_name(registry_uri: &str) -> String { format!(