Skip to content

Commit ec9c451

Browse files
author
Sylvain MARIE
committed
Migration to latest CI/CD and build template
1 parent 3c4da96 commit ec9c451

20 files changed

Lines changed: 1549 additions & 549 deletions

.github/workflows/base.yml

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
# .github/workflows/base.yml
2+
name: Build
3+
on:
4+
# this one is to trigger the workflow manually from the interface
5+
# workflow_dispatch:
6+
7+
push:
8+
tags:
9+
- '*'
10+
branches:
11+
- main
12+
pull_request:
13+
branches:
14+
- main
15+
jobs:
16+
# pre-job to read nox tests matrix - see https://stackoverflow.com/q/66747359/7262247
17+
list_nox_test_sessions:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- uses: actions/checkout@v2
21+
- uses: actions/setup-python@v1
22+
with:
23+
python-version: 3.7
24+
architecture: x64
25+
26+
- name: Install noxfile requirements
27+
shell: bash -l {0}
28+
run: pip install -r noxfile-requirements.txt
29+
30+
- name: List 'tests' nox sessions
31+
id: set-matrix
32+
run: echo "::set-output name=matrix::$(nox -s gha_list -- tests)"
33+
outputs:
34+
matrix: ${{ steps.set-matrix.outputs.matrix }} # save nox sessions list to outputs
35+
36+
run_all_tests:
37+
needs: list_nox_test_sessions
38+
strategy:
39+
fail-fast: false
40+
matrix:
41+
os: [ ubuntu-latest ] # , macos-latest, windows-latest]
42+
# all nox sessions: manually > dynamically from previous job
43+
# nox_session: ["tests-2.7", "tests-3.7"]
44+
nox_session: ${{ fromJson(needs.list_nox_test_sessions.outputs.matrix) }}
45+
46+
name: ${{ matrix.os }} ${{ matrix.nox_session }} # ${{ matrix.name_suffix }}
47+
runs-on: ${{ matrix.os }}
48+
steps:
49+
- uses: actions/checkout@v2
50+
51+
# Conda install
52+
- name: Install conda v3.7
53+
uses: conda-incubator/setup-miniconda@v2
54+
with:
55+
# auto-update-conda: true
56+
python-version: 3.7
57+
activate-environment: noxenv
58+
- run: conda info
59+
shell: bash -l {0} # so that conda works
60+
- run: conda list
61+
shell: bash -l {0} # so that conda works
62+
63+
# Nox install + run
64+
- name: Install noxfile requirements
65+
shell: bash -l {0} # so that conda works
66+
run: pip install -r noxfile-requirements.txt
67+
- run: conda list
68+
shell: bash -l {0} # so that conda works
69+
- run: nox -s "${{ matrix.nox_session }}"
70+
shell: bash -l {0} # so that conda works
71+
72+
# Share ./docs/reports so that they can be deployed with doc in next job
73+
- name: Share reports with other jobs
74+
# if: matrix.nox_session == '...': not needed, if empty wont be shared
75+
uses: actions/upload-artifact@master
76+
with:
77+
name: reports_dir
78+
path: ./docs/reports
79+
80+
publish_release:
81+
needs: run_all_tests
82+
runs-on: ubuntu-latest
83+
if: github.event_name == 'push'
84+
steps:
85+
- name: GitHub context to debug conditional steps
86+
env:
87+
GITHUB_CONTEXT: ${{ toJSON(github) }}
88+
run: echo "$GITHUB_CONTEXT"
89+
90+
- uses: actions/checkout@v2
91+
with:
92+
fetch-depth: 0 # so that gh-deploy works
93+
94+
# 1) retrieve the reports generated previously
95+
- name: Retrieve reports
96+
uses: actions/download-artifact@master
97+
with:
98+
name: reports_dir
99+
path: ./docs/reports
100+
101+
# Conda install
102+
- name: Install conda v3.7
103+
uses: conda-incubator/setup-miniconda@v2
104+
with:
105+
# auto-update-conda: true
106+
python-version: 3.7
107+
activate-environment: noxenv
108+
- run: conda info
109+
shell: bash -l {0} # so that conda works
110+
- run: conda list
111+
shell: bash -l {0} # so that conda works
112+
113+
# Nox install
114+
- name: Install noxfile requirements
115+
shell: bash -l {0} # so that conda works
116+
run: pip install -r noxfile-requirements.txt
117+
- run: conda list
118+
shell: bash -l {0} # so that conda works
119+
120+
# 5) Run the flake8 report and badge
121+
- name: Run flake8 analysis and generate corresponding badge
122+
shell: bash -l {0} # so that conda works
123+
run: nox -s flake8
124+
125+
# -------------- only on Ubuntu + MAIN PUSH (no pull request, no tag) -----------
126+
127+
# 5) Publish the doc and test reports
128+
- name: \[not on TAG\] Publish documentation, tests and coverage reports
129+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads') # startsWith(matrix.os,'ubuntu')
130+
shell: bash -l {0} # so that conda works
131+
run: nox -s publish
132+
133+
# 6) Publish coverage report
134+
- name: \[not on TAG\] Create codecov.yaml with correct paths
135+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads')
136+
shell: bash
137+
run: |
138+
cat << EOF > codecov.yml
139+
# codecov.yml
140+
fixes:
141+
- "/home/runner/work/smarie/python-yamlable/::" # Correct paths
142+
EOF
143+
- name: \[not on TAG\] Publish coverage report
144+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads')
145+
uses: codecov/codecov-action@v1
146+
with:
147+
files: ./docs/reports/coverage/coverage.xml
148+
149+
# -------------- only on Ubuntu + TAG PUSH (no pull request) -----------
150+
151+
# 7) Create github release and build the wheel
152+
- name: \[TAG only\] Build wheel and create github release
153+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
154+
shell: bash -l {0} # so that conda works
155+
run: nox -s release -- ${{ secrets.GITHUB_TOKEN }}
156+
157+
# 8) Publish the wheel on PyPi
158+
- name: \[TAG only\] Deploy on PyPi
159+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
160+
uses: pypa/gh-action-pypi-publish@release/v1
161+
with:
162+
user: __token__
163+
password: ${{ secrets.PYPI_API_TOKEN }}
164+
165+
delete-artifacts:
166+
needs: publish_release
167+
runs-on: ubuntu-latest
168+
if: github.event_name == 'push'
169+
steps:
170+
- uses: kolpav/purge-artifacts-action@v1
171+
with:
172+
token: ${{ secrets.GITHUB_TOKEN }}
173+
expire-in: 0 # Setting this to 0 will delete all artifacts

.gitignore

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ parts/
2020
sdist/
2121
var/
2222
wheels/
23+
pip-wheel-metadata/
24+
share/python-wheels/
2325
*.egg-info/
2426
.installed.cfg
2527
*.egg
@@ -38,14 +40,17 @@ pip-delete-this-directory.txt
3840
# Unit test / coverage reports
3941
htmlcov/
4042
.tox/
43+
.nox/
4144
.coverage
4245
.coverage.*
4346
.cache
4447
nosetests.xml
4548
coverage.xml
4649
*.cover
50+
*.py,cover
4751
.hypothesis/
4852
.pytest_cache/
53+
*/_version.py
4954

5055
# Translations
5156
*.mo
@@ -55,6 +60,7 @@ coverage.xml
5560
*.log
5661
local_settings.py
5762
db.sqlite3
63+
db.sqlite3-journal
5864

5965
# Flask stuff:
6066
instance/
@@ -72,11 +78,26 @@ target/
7278
# Jupyter Notebook
7379
.ipynb_checkpoints
7480

81+
# IPython
82+
profile_default/
83+
ipython_config.py
84+
7585
# pyenv
7686
.python-version
7787

78-
# celery beat schedule file
88+
# pipenv
89+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
90+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
91+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
92+
# install all needed dependencies.
93+
#Pipfile.lock
94+
95+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
96+
__pypackages__/
97+
98+
# Celery stuff
7999
celerybeat-schedule
100+
celerybeat.pid
80101

81102
# SageMath parsed files
82103
*.sage.py
@@ -85,7 +106,7 @@ celerybeat-schedule
85106
.env
86107
.venv
87108
env/
88-
venv/
109+
venv*/
89110
ENV/
90111
env.bak/
91112
venv.bak/
@@ -102,13 +123,20 @@ venv.bak/
102123

103124
# mypy
104125
.mypy_cache/
126+
.dmypy.json
127+
dmypy.json
128+
129+
# Pyre type checker
130+
.pyre/
131+
132+
# PyCharm development
133+
/.idea
105134

106-
# Pycharm
107-
.idea/
135+
# OSX
136+
.DS_Store
108137

109-
# Mkdocs
110-
site/
138+
# JUnit and coverage reports
139+
docs/reports
111140

112-
# travis CI
113-
github_travis_rsa*
114-
reports
141+
# ODSClient cache
142+
.odsclient

README.md

Lines changed: 46 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
*Convert Python objects to YAML and back.*
44

5-
[![Python versions](https://img.shields.io/pypi/pyversions/yamlable.svg)](https://pypi.python.org/pypi/yamlable/) [![Build Status](https://travis-ci.org/smarie/python-yamlable.svg?branch=master)](https://travis-ci.org/smarie/python-yamlable) [![Tests Status](https://smarie.github.io/python-yamlable/junit/junit-badge.svg?dummy=8484744)](https://smarie.github.io/python-yamlable/junit/report.html) [![codecov](https://codecov.io/gh/smarie/python-yamlable/branch/master/graph/badge.svg)](https://codecov.io/gh/smarie/python-yamlable)
5+
[![Python versions](https://img.shields.io/pypi/pyversions/yamlable.svg)](https://pypi.python.org/pypi/yamlable/) [![Build Status](https://github.com/smarie/python-yamlable/actions/workflows/base.yml/badge.svg)](https://github.com/smarie/python-yamlable/actions/workflows/base.yml) [![Tests Status](https://smarie.github.io/python-yamlable/reports/junit/junit-badge.svg?dummy=8484744)](https://smarie.github.io/python-yamlable/reports/junit/report.html) [![Coverage Status](https://smarie.github.io/python-yamlable/reports/coverage/coverage-badge.svg?dummy=8484744)](https://smarie.github.io/python-yamlable/reports/coverage/index.html) [![Flake8 Status](https://smarie.github.io/python-yamlable/reports/flake8/flake8-badge.svg?dummy=8484744)](https://smarie.github.io/python-yamlable/reports/flake8/index.html)
66

7-
[![Documentation](https://img.shields.io/badge/doc-latest-blue.svg)](https://smarie.github.io/python-yamlable/) [![PyPI](https://img.shields.io/pypi/v/yamlable.svg)](https://pypi.python.org/pypi/yamlable/) [![Downloads](https://pepy.tech/badge/yamlable)](https://pepy.tech/project/yamlable) [![Downloads per week](https://pepy.tech/badge/yamlable/week)](https://pepy.tech/project/yamlable) [![GitHub stars](https://img.shields.io/github/stars/smarie/python-yamlable.svg)](https://github.com/smarie/python-yamlable/stargazers)
7+
[![Documentation](https://img.shields.io/badge/doc-latest-blue.svg)](https://smarie.github.io/python-yamlable/) [![PyPI](https://img.shields.io/pypi/v/yamlable.svg)](https://pypi.python.org/pypi/yamlable/) [![Downloads](https://pepy.tech/badge/yamlable)](https://pepy.tech/project/yamlable) [![Downloads per week](https://pepy.tech/badge/yamlable/week)](https://pepy.tech/project/yamlable) [![GitHub stars](https://img.shields.io/github/stars/smarie/python-yamlable.svg)](https://github.com/smarie/python-yamlable/stargazers) [![codecov](https://codecov.io/gh/smarie/python-yamlable/branch/main/graph/badge.svg)](https://codecov.io/gh/smarie/python-yamlable)
88

99
**This is the readme for developers.** The documentation for users is available here: [https://smarie.github.io/python-yamlable/](https://smarie.github.io/python-yamlable/)
1010

@@ -14,66 +14,75 @@ Contributions are welcome ! Simply fork this project on github, commit your cont
1414

1515
Here is a non-exhaustive list of interesting open topics: [https://github.com/smarie/python-yamlable/issues](https://github.com/smarie/python-yamlable/issues)
1616

17-
## Running the tests
17+
## `nox` setup
1818

19-
This project uses `pytest`.
19+
This project uses `nox` to define all lifecycle tasks. In order to be able to run those tasks, you should create python 3.7 environment and install the requirements:
2020

2121
```bash
22-
pytest -v yamlable/tests/
22+
>>> conda create -n noxenv python="3.7"
23+
>>> activate noxenv
24+
(noxenv) >>> pip install -r noxfile-requirements.txt
2325
```
2426

25-
You may need to install requirements for setup beforehand, using
27+
You should then be able to list all available tasks using:
2628

27-
```bash
28-
pip install -r ci_tools/requirements-test.txt
29+
```
30+
>>> nox --list
31+
Sessions defined in <path>\noxfile.py:
32+
33+
* tests-2.7 -> Run the test suite, including test reports generation and coverage reports.
34+
* tests-3.5 -> Run the test suite, including test reports generation and coverage reports.
35+
* tests-3.6 -> Run the test suite, including test reports generation and coverage reports.
36+
* tests-3.8 -> Run the test suite, including test reports generation and coverage reports.
37+
* tests-3.7 -> Run the test suite, including test reports generation and coverage reports.
38+
- docs-3.7 -> Generates the doc and serves it on a local http server. Pass '-- build' to build statically instead.
39+
- publish-3.7 -> Deploy the docs+reports on github pages. Note: this rebuilds the docs
40+
- release-3.7 -> Create a release on github corresponding to the latest tag
2941
```
3042

31-
## Packaging
43+
## Running the tests and generating the reports
3244

33-
This project uses `setuptools_scm` to synchronise the version number. Therefore the following command should be used for development snapshots as well as official releases:
45+
This project uses `pytest` so running `pytest` at the root folder will execute all tests on current environment. However it is a bit cumbersome to manage all requirements by hand ; it is easier to use `nox` to run `pytest` on all supported python environments with the correct package requirements:
3446

3547
```bash
36-
python setup.py egg_info bdist_wheel rotate -m.whl -k3
48+
nox
3749
```
3850

39-
You may need to install requirements for setup beforehand, using
51+
Tests and coverage reports are automatically generated under `./docs/reports` for one of the sessions (`tests-3.7`).
4052

41-
```bash
42-
pip install -r ci_tools/requirements-setup.txt
43-
```
44-
45-
## Generating the documentation page
53+
If you wish to execute tests on a specific environment, use explicit session names, e.g. `nox -s tests-3.6`.
4654

47-
This project uses `mkdocs` to generate its documentation page. Therefore building a local copy of the doc page may be done using:
4855

49-
```bash
50-
mkdocs build -f docs/mkdocs.yml
51-
```
56+
## Editing the documentation
5257

53-
You may need to install requirements for doc beforehand, using
58+
This project uses `mkdocs` to generate its documentation page. Therefore building a local copy of the doc page may be done using `mkdocs build -f docs/mkdocs.yml`. However once again things are easier with `nox`. You can easily build and serve locally a version of the documentation site using:
5459

5560
```bash
56-
pip install -r ci_tools/requirements-doc.txt
61+
>>> nox -s docs
62+
nox > Running session docs-3.7
63+
nox > Creating conda env in .nox\docs-3-7 with python=3.7
64+
nox > [docs] Installing requirements with pip: ['mkdocs-material', 'mkdocs', 'pymdown-extensions', 'pygments']
65+
nox > python -m pip install mkdocs-material mkdocs pymdown-extensions pygments
66+
nox > mkdocs serve -f ./docs/mkdocs.yml
67+
INFO - Building documentation...
68+
INFO - Cleaning site directory
69+
INFO - The following pages exist in the docs directory, but are not included in the "nav" configuration:
70+
- long_description.md
71+
INFO - Documentation built in 1.07 seconds
72+
INFO - Serving on http://127.0.0.1:8000
73+
INFO - Start watching changes
74+
...
5775
```
5876

59-
## Generating the test reports
60-
61-
The following commands generate the html test report and the associated badge.
77+
While this is running, you can edit the files under `./docs/` and browse the automatically refreshed documentation at the local [http://127.0.0.1:8000](http://127.0.0.1:8000) page.
6278

63-
```bash
64-
pytest --junitxml=junit.xml -v yamlable/tests/
65-
ant -f ci_tools/generate-junit-html.xml
66-
python ci_tools/generate-junit-badge.py
67-
```
79+
Once you are done, simply hit `<CTRL+C>` to stop the session.
6880

69-
### PyPI Releasing memo
81+
Publishing the documentation (including tests and coverage reports) is done automatically by [the continuous integration engine](https://github.com/smarie/python-yamlable/actions), using the `nox -s publish` session, this is not needed for local development.
7082

71-
This project is now automatically deployed to PyPI when a tag is created. Anyway, for manual deployment we can use:
83+
## Packaging
7284

73-
```bash
74-
twine upload dist/* -r pypitest
75-
twine upload dist/*
76-
```
85+
This project uses `setuptools_scm` to synchronise the version number. Therefore the following command should be used for development snapshots as well as official releases: `python setup.py sdist bdist_wheel`. However this is not generally needed since [the continuous integration engine](https://github.com/smarie/python-yamlable/actions) does it automatically for us on git tags. For reference, this is done in the `nox -s release` session.
7786

7887
### Merging pull requests with edits - memo
7988

0 commit comments

Comments
 (0)