Skip to content

Commit 711636b

Browse files
alecbcsgithub-actions[bot]
alecbcs
and
github-actions[bot]
authored
ci: enable output generation for tutorial (#465)
* ci: enable output generation for tutorial * Mount entire spack home directory for cache reuse * Update generated outputs on behalf of [@alecbcs] --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent 8d7d8b8 commit 711636b

100 files changed

Lines changed: 12696 additions & 5406 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,17 @@ jobs:
5050
needs: changes
5151
uses: ./.github/workflows/container.yml
5252

53-
# outputs:
54-
# if: |
55-
# !cancelled()
56-
# && contains(needs.*.result, 'success')
57-
# && !contains(needs.*.result, 'failure')
58-
# && (needs.changes.outputs.outputs == 'true'
59-
# || startsWith(github.ref, 'refs/tags/'))
60-
# needs: [ changes, container ]
61-
# uses: ./.github/workflows/outputs.yml
53+
outputs:
54+
if: |
55+
!cancelled()
56+
&& contains(needs.*.result, 'success')
57+
&& !contains(needs.*.result, 'failure')
58+
&& (needs.changes.outputs.outputs == 'true'
59+
|| startsWith(github.ref, 'refs/tags/'))
60+
needs: [ changes, container ]
61+
uses: ./.github/workflows/outputs.yml
62+
with:
63+
with_pr_container: ${{ needs.container.result == 'success' }}
6264

6365
style:
6466
if: ${{ needs.changes.outputs.style == 'true' }}

‎.github/workflows/container.yml‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,21 +10,6 @@ jobs:
1010
packages: write
1111

1212
steps:
13-
- name: Remove pre-installed apps to make space for build
14-
run: |
15-
sudo apt-get remove -y '^llvm-.*'
16-
sudo apt-get remove -y '^dotnet-.*'
17-
sudo apt-get remove -y 'php.*'
18-
sudo apt-get remove -y '^mongodb-.*'
19-
sudo apt-get remove -y '^mysql-.*'
20-
sudo apt-get autoremove -y
21-
sudo apt-get clean
22-
sudo rm -rf /usr/share/dotnet/
23-
sudo rm -rf /usr/local/share/powershell
24-
sudo rm -rf /usr/local/share/chromium
25-
sudo rm -rf /usr/local/lib/android
26-
sudo rm -rf /usr/local/lib/node_modules
27-
2813
- name: Set up QEMU
2914
uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
3015

‎.github/workflows/outputs.yml‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
name: Generate outputs
22
on:
33
workflow_call:
4+
inputs:
5+
with_pr_container:
6+
required: true
7+
type: boolean
48

59
jobs:
610
re-generate:
@@ -10,22 +14,6 @@ jobs:
1014
contents: write
1115

1216
steps:
13-
- name: Remove pre-installed apps to make space for build
14-
run: |
15-
sudo apt-get remove -y '^llvm-.*'
16-
sudo apt-get remove -y '^dotnet-.*'
17-
sudo apt-get remove -y 'php.*'
18-
sudo apt-get remove -y '^mongodb-.*'
19-
sudo apt-get remove -y '^mysql-.*'
20-
sudo apt-get autoremove -y
21-
sudo apt-get clean
22-
sudo rm -rf /usr/share/dotnet/
23-
sudo rm -rf /usr/local/share/powershell
24-
sudo rm -rf /usr/local/share/chromium
25-
sudo rm -rf /usr/local/lib/android
26-
sudo rm -rf /usr/local/lib/node_modules
27-
sudo docker system prune -af
28-
2917
- name: Checkout (Push)
3018
if: github.event_name != 'pull_request'
3119
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
@@ -40,32 +28,34 @@ jobs:
4028
repository: ${{ github.event.pull_request.head.repo.full_name }}
4129

4230
- name: Download container from artifact if PR
43-
if: github.event_name == 'pull_request' && needs.container.result == 'success'
31+
if: github.event_name == 'pull_request' && inputs.with_pr_container
4432
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
4533
with:
4634
name: tutorial-container
4735
path: /tmp
4836

4937
- name: Import container from PR artifact
50-
if: github.event_name == 'pull_request' && needs.container.result == 'success'
38+
if: github.event_name == 'pull_request' && inputs.with_pr_container
5139
run: |
5240
docker load --input /tmp/tutorial-container.tar
5341
docker image ls -a
5442
5543
- name: Re-generate outputs from tutorial scripts
5644
run: |
5745
chmod -R a+w outputs
58-
make -C outputs -j $(nproc) DOCKER_RUN_OPTS="-e COLUMNS=94 -e COLIFY_SIZE=24x94" container=ghcr.io/spack/tutorial
46+
mkdir spack-home && sudo chown -R 1000:1000 spack-home
47+
DOCKER_RUN_OPTS="-e COLUMNS=94 -e COLIFY_SIZE=24x94 -v $(pwd)/spack-home:/home/spack"
48+
make -C outputs -j $(nproc) DOCKER_RUN_OPTS="$DOCKER_RUN_OPTS" container=ghcr.io/spack/tutorial
5949
6050
- name: Show diff of modified files
6151
run: |
62-
git diff HEAD .
52+
git diff HEAD outputs/
6353
6454
- name: Push modified files to branch
6555
if: ( github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork )
6656
run: |
6757
git config user.name "github-actions[bot]"
6858
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
69-
git add .
59+
git add outputs/
7060
git commit -m "Update generated outputs on behalf of [@${{ github.actor }}]"
7161
git push

‎container/Dockerfile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ ENV DEBIAN_FRONTEND=noninteractive
55
# Install AWS cli
66
RUN apt-get update -y && \
77
apt-get install -y --no-install-recommends \
8-
awscli \
98
bash-completion \
9+
ca-certificates \
1010
emacs \
1111
jq \
1212
less \
13-
vim
13+
vim \
14+
rclone
1415

1516
# Remove GCC 12 from container
1617
RUN apt-get remove -y gcc-12
1718

1819
# Download the buildcache
1920
RUN mkdir /mirror
20-
RUN aws --region us-east-1 --no-sign-request s3 sync --only-show-errors s3://spack-binaries/releases/v0.23/tutorial /mirror
21+
RUN rclone copy :s3:spack-binaries/v0.23.1/tutorial /mirror
2122

2223
RUN chmod -R go+r /mirror
2324

‎outputs/Makefile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ sections := environments
2525
run_targets := $(addprefix run-,$(sections))
2626

2727
# Entrypoint
28-
update-outputs: #run
28+
update-outputs: run
2929
echo "Filtering raw outputs though col"
3030
for raw in raw/*/*.out; do \
3131
out=$$(echo $$raw | sed 's.raw/..'); \
@@ -34,17 +34,17 @@ update-outputs: #run
3434

3535
run: run-scripting
3636

37-
run-scripting: run-cache scripting.sh
38-
run-cache: run-dev cache.sh
37+
run-scripting: run-dev scripting.sh
38+
# run-cache: run-dev cache.sh
3939
run-dev: run-packaging dev.sh
4040
run-packaging: run-stacks packaging.sh
4141
run-stacks: run-environments stacks.sh
4242
run-environments: run-basics environments.sh
4343
run-basics: basics.sh init_spack.sh defs.sh
4444

4545
local: local-scripting
46-
local-scripting: local-cache scripting.sh
47-
local-cache: local-dev cache.sh
46+
local-scripting: local-dev scripting.sh
47+
# local-cache: local-dev cache.sh
4848
local-dev: local-packaging dev.sh
4949
local-packaging: local-stacks packaging.sh
5050
local-stacks: local-environments stacks.sh

‎outputs/basics/clone.out‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
$ git clone --depth=2 --branch=releases/v0.23 https://github.com/spack/spack.git ~/spack
2-
Cloning into '/home/spack5/spack'...
3-
remote: Enumerating objects: 20970, done.K
4-
remote: Counting objects: 100% (20970/20970), done.K
5-
remote: Compressing objects: 100% (11467/11467), done.K
6-
Receiving objects: 100% (20970/20970), 14.78 MiB | 23.43 MiB/s, done.
7-
remote: Total 20970 (delta 1785), reused 14006 (delta 1489), pack-reused 0 (from 0)K
8-
Resolving deltas: 100% (1785/1785), done.
2+
Cloning into '/home/spack/spack'...
3+
remote: Enumerating objects: 20971, done.K
4+
remote: Counting objects: 100% (20971/20971), done.K
5+
remote: Compressing objects: 100% (11448/11448), done.K
6+
remote:nTotale20971 (delta01801),9reused014170i(delta.1509),/pack-reused 0 (from 0)K
7+
Receiving objects: 100% (20971/20971), 14.78 MiB | 20.70 MiB/s, done.
8+
Resolving deltas: 100% (1801/1801), done.
99
$ cd ~/spack

‎outputs/basics/compiler-add-location.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
$ spack compiler add "$(spack location -i gcc@12)"
2-
==> Added 1 new compiler to /home/spack5/.spack/linux/compilers.yaml
2+
==> Added 1 new compiler to /home/spack/.spack/linux/compilers.yaml
33
gcc@12.3.0
44
==> Compilers are defined in the following files:
5-
/home/spack5/.spack/linux/compilers.yaml
5+
/home/spack/.spack/linux/compilers.yaml

‎outputs/basics/find-lf.out‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@ $ spack find -lf
33
xvsfmyv glibc@2.35%clang gcbcod5 gmake@4.4.1%clang jjykcjt zlib-ng@2.2.1%clang
44

55
-- linux-ubuntu22.04-x86_64_v3 / gcc@10.5.0 ---------------------
6-
g7dc4ou gcc-runtime@10.5.0%gcc tcngwuv glibc@2.35%gcc he2qdcb gmake@4.4.1%gcc aiapvpp zlib-ng@2.2.1%gcc
6+
g7dc4ou gcc-runtime@10.5.0%gcc he2qdcb gmake@4.4.1%gcc
7+
tcngwuv glibc@2.35%gcc aiapvpp zlib-ng@2.2.1%gcc
78

89
-- linux-ubuntu22.04-x86_64_v3 / gcc@11.4.0 ---------------------
9-
hshzy76 gcc-runtime@11.4.0%gcc a7drdl4 glibc@2.35%gcc srkzfjr gmake@4.4.1%gcc ocxrmff zlib-ng@2.0.7%gcc umrbkwv zlib-ng@2.0.7%gcc cflags=-O3
10+
hshzy76 gcc-runtime@11.4.0%gcc ocxrmff zlib-ng@2.0.7%gcc
11+
a7drdl4 glibc@2.35%gcc umrbkwv zlib-ng@2.0.7%gcc cflags=-O3
12+
srkzfjr gmake@4.4.1%gcc
1013
==> 12 installed packages

‎outputs/basics/find-p-gcc.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
$ spack find -p gcc
22
-- linux-ubuntu22.04-x86_64_v3 / gcc@11.4.0 ---------------------
3-
gcc@12.3.0 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/gcc-12.3.0-jjgfqpdmihb2cq5ca7txia5oshruq56q
3+
gcc@12.3.0 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/gcc-12.3.0-jjgfqpdmihb2cq5ca7txia5oshruq56q
44
==> 1 installed package

‎outputs/basics/find-px.out‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
$ spack find -px
22
-- linux-ubuntu22.04-x86_64_v3 / clang@14.0.0 -------------------
3-
zlib-ng@2.2.1 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/clang-14.0.0/zlib-ng-2.2.1-jjykcjtfyoezsckykaklun6a75bzd24o
3+
zlib-ng@2.2.1 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/clang-14.0.0/zlib-ng-2.2.1-jjykcjtfyoezsckykaklun6a75bzd24o
44

55
-- linux-ubuntu22.04-x86_64_v3 / gcc@11.4.0 ---------------------
6-
gmake@4.4.1 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/gmake-4.4.1-srkzfjru5z55bl5gy24dbjsrvqwg5fom
7-
hdf5@1.14.5 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/hdf5-1.14.5-ckoh67fpzb5gn7mti2xzruvlxd424par
8-
hdf5@1.14.5 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/hdf5-1.14.5-6ddnwvambfcyhhhs7ehog6gkftrkljnt
9-
hdf5@1.14.5 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/hdf5-1.14.5-tgtmflo4s6hqenvkle7urqiz2w6rumph
10-
tcl@8.6.12 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/tcl-8.6.12-imrm45j44xmboxdqpj7akq2lwkwza4vo
11-
tcl@8.6.12 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/tcl-8.6.12-w4gwr4ezmenbvt7u5zdyjdkfzapsz5lt
12-
trilinos@16.0.0 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/trilinos-16.0.0-63sfm7p6vebucsnbmwfnx36hiyal7e3e
13-
zlib-ng@2.0.7 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/zlib-ng-2.0.7-ocxrmffc7qnibijshhygluei5qiycvja
14-
zlib-ng@2.0.7 /home/spack5/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/zlib-ng-2.0.7-umrbkwvnydtpljzy4isoednibe4mwcvz
6+
gmake@4.4.1 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/gmake-4.4.1-srkzfjru5z55bl5gy24dbjsrvqwg5fom
7+
hdf5@1.14.5 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/hdf5-1.14.5-ckoh67fpzb5gn7mti2xzruvlxd424par
8+
hdf5@1.14.5 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/hdf5-1.14.5-6ddnwvambfcyhhhs7ehog6gkftrkljnt
9+
hdf5@1.14.5 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/hdf5-1.14.5-tgtmflo4s6hqenvkle7urqiz2w6rumph
10+
tcl@8.6.12 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/tcl-8.6.12-imrm45j44xmboxdqpj7akq2lwkwza4vo
11+
tcl@8.6.12 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/tcl-8.6.12-w4gwr4ezmenbvt7u5zdyjdkfzapsz5lt
12+
trilinos@16.0.0 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/trilinos-16.0.0-63sfm7p6vebucsnbmwfnx36hiyal7e3e
13+
zlib-ng@2.0.7 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/zlib-ng-2.0.7-ocxrmffc7qnibijshhygluei5qiycvja
14+
zlib-ng@2.0.7 /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/zlib-ng-2.0.7-umrbkwvnydtpljzy4isoednibe4mwcvz
1515
==> 10 installed packages

0 commit comments

Comments
 (0)