Project updated to latest build process and badges

Author: Sylvain MARIE

.travis.yml

@@ -2,41 +2,38 @@ language: python
 
 cache: pip
 
-python:
-#  - "2.6"
-  - "2.7"
-#  - "3.2"
-#  - "3.3"
-#  - "3.4"
-  - "3.5"
-#  - "3.5-dev" # 3.5 development branch
-  - "3.6"
-#  - "3.6-dev" # 3.6 development branch
-#  - "3.7"
-#  - "3.7-dev" # 3.7 development branch
-#  - "nightly" # currently points to 3.7-dev
-# PyPy versions
-#  - "pypy"  # PyPy2 2.5.0
-#  - "pypy3" # Pypy3 2.4.0
-#  - "pypy-5.3.1"
-#
+matrix:
+  fast_finish: true
+  include:
+#    - python: 2.6
+    - python: 2.7
+#    - python: 3.3
+#    - python: 3.4
+    - python: 3.5
+    - python: 3.6
+    - python: 3.7
+      dist: xenial
+      sudo: true
 
 env:
   global:
   - GH_REF: git@github.com:smarie/python-yamlable.git
 
 before_install:
+  # (a) linux dependencies
   - sudo apt-get install pandoc
   - sudo apt-get install ant
   - sudo apt-get install ant-optional
 
 install:
-  - pip install -r ci_tools/requirements-setup.txt
-  - pip install -r ci_tools/requirements-test.txt
-  - pip install -r ci_tools/requirements-report.txt
-  - pip install -r ci_tools/requirements-doc.txt
-  - pip install codecov    # https://github.com/codecov/example-python. This is specific to travis integration
-# - pip install coveralls  # this is an alternative to codecov
+  - pip list
+  # needs to be installed beforehand
+  - pip install setuptools_scm
+  - python ci_tools/py_install.py pip ci_tools/requirements-pip.txt
+  # travis-specific installs
+  - pip install PyGithub  # for ci_tools/github_release.py
+  - pip install codecov  # See https://github.com/codecov/example-python.
+  - pip list
 
 script:
 #  - coverage run tests.py
@@ -48,14 +45,14 @@ script:
 # now done in a dedicated script to capture exit code 1 and transform it to 0
   - chmod a+x ./ci_tools/run_tests.sh
   - sh ./ci_tools/run_tests.sh
+  - python ci_tools/generate-junit-badge.py 100  # generates the badge for the test results and fail build if less than x%
 
 after_success:
 # ***reporting***
 #  - junit2html junit.xml testrun.html  output is really not nice
   - ant -f ci_tools/generate-junit-html.xml  # generates the html for the test results. Actually we dont use it anymore
-  - python ci_tools/generate-junit-badge.py  # generates the badge for the test results
   - codecov
-  - pylint yamlable  # note that at the moment the report is simply lost, we dont transform the result into anything
+  #- pylint yamlable  # note that at the moment the report is simply lost, we dont transform the result into anything
 # ***documentation***
   - mkdocs build -f docs/mkdocs.yml
   - mv reports/junit site/
@@ -67,11 +64,11 @@ after_success:
     if [ -s "ci_tools/github_travis_rsa" ]; then
       chmod 600 ci_tools/github_travis_rsa
       eval `ssh-agent -s`  # launch the authentication agent
-      ssh-add ci_tools/github_travis_rsa  # register the key
+      ssh-add ci_tools/github_travis_rsa  # register the decrypted key
       git config user.name "Automatic Publish"
       git config user.email "sylvain.marie@schneider-electric.com"
       git remote add gh-remote "${GH_REF}";
-      git fetch gh-remote && git fetch gh-remote gh-pages:gh-pages;
+      git fetch gh-remote && git fetch gh-remote gh-pages:gh-pages;  # make sure we have the latest gh-remote
       # push but only if this is not a build triggered by a pull request
       # note: here we use the --dirty flag so that mkdocs does not clean the additional reports that we copied in the site
       if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${TRAVIS_PYTHON_VERSION}" = "3.5" ]; then echo "Pushing to github"; PYTHONPATH=yamlable/ mkdocs gh-deploy -v --dirty -f docs/mkdocs.yml --remote-name gh-remote; git push gh-remote gh-pages; fi;
@@ -80,18 +77,24 @@ after_success:
     fi
 
 deploy:
-  provider: pypi
-  user: "smarie"
-  password:
-    secure: "TLBvYaFRplEFa8T0AW5b0LnRnz19/k/BEjtiiw4aWWrXTfNiWW1RRREG1iqqsqa6rkv547BVug2zj0SeeLvoncjGYclh92bmKMFhLVOlJtmYCIp8pNXT/3fEtrZosUN2OX4Z6qfOWnnefwjBB19rl18XPq/6zTyZ9dwKjkAfN+PPkHLqPtQ22+TcUe7f9jkDOs1pR0afacHz3HuBE5iaWbcDD2yCteJbfHwpFExfyO/X6LcjtQu63KH2NGvFP9qvwGrtnDIvmc1sWW9zi85H/j9T1TLQovJrY0oS8F+QOXbFlz/mOpORnh2sUj3f/nye0CvutnAqgS2OcRMqEAMIrdWzQOu8aBkKQuTOtiP+YKv7q7+GbWDIHJZK+whCWscS6XORVIaP5yGHc6SNZQqXarYskJFuRyJoClL51SxNsUzN4+yOLsdAqI/lvK2jIx4c2IztxuHEmbNwg1/R5/3YTk8g48U/nqOuIWBTNr0YwRRL/lBCxzGfHSjfcEnxzB0h/ORn0wBXig6tmNRyFAeA3qMHRekUgHjmzdZWw1yty3j1+sxdhOJXge+/AvqnNhSdY06w12WhEi4MV7p2pS5ZMX3Wnxvla/AzkCM4hrXxTYycAgFYjU2WkCCYqJwfV+Noa2hDbKPMHucoYBfB+qYNHUu0f+5uEGY3V5EhxZCmrGE="
-  on:
-    tags: true
-    python: 3.5  #only one of the builds have to be deployed
-#  server: https://test.pypi.org/legacy/
-  distributions: "sdist bdist_wheel"
+  # Deploy on PyPI on tags
+  - provider: pypi
+    user: "smarie"
+    password:
+      secure: "TLBvYaFRplEFa8T0AW5b0LnRnz19/k/BEjtiiw4aWWrXTfNiWW1RRREG1iqqsqa6rkv547BVug2zj0SeeLvoncjGYclh92bmKMFhLVOlJtmYCIp8pNXT/3fEtrZosUN2OX4Z6qfOWnnefwjBB19rl18XPq/6zTyZ9dwKjkAfN+PPkHLqPtQ22+TcUe7f9jkDOs1pR0afacHz3HuBE5iaWbcDD2yCteJbfHwpFExfyO/X6LcjtQu63KH2NGvFP9qvwGrtnDIvmc1sWW9zi85H/j9T1TLQovJrY0oS8F+QOXbFlz/mOpORnh2sUj3f/nye0CvutnAqgS2OcRMqEAMIrdWzQOu8aBkKQuTOtiP+YKv7q7+GbWDIHJZK+whCWscS6XORVIaP5yGHc6SNZQqXarYskJFuRyJoClL51SxNsUzN4+yOLsdAqI/lvK2jIx4c2IztxuHEmbNwg1/R5/3YTk8g48U/nqOuIWBTNr0YwRRL/lBCxzGfHSjfcEnxzB0h/ORn0wBXig6tmNRyFAeA3qMHRekUgHjmzdZWw1yty3j1+sxdhOJXge+/AvqnNhSdY06w12WhEi4MV7p2pS5ZMX3Wnxvla/AzkCM4hrXxTYycAgFYjU2WkCCYqJwfV+Noa2hDbKPMHucoYBfB+qYNHUu0f+5uEGY3V5EhxZCmrGE="
+    on:
+      tags: true
+      python: 3.5  #only one of the builds have to be deployed
+    # server: https://test.pypi.org/legacy/
+    distributions: "sdist bdist_wheel"
 
-matrix:
-    fast_finish: true
+  # Create a github release on tags
+  - provider: script
+    script: python ci_tools/github_release.py -s $GITHUB_TOKEN --repo-slug smarie/python-yamlable -cf ./docs/changelog.md -d https://smarie.github.io/python-yamlable/changelog/ $TRAVIS_TAG
+    skip_cleanup: true
+    on:
+      tags: true
+      python: 3.5  #only one of the builds have to be deployed
 
 notifications:
   email:

README.md

@@ -2,7 +2,9 @@
 
 *Convert Python objects to YAML and back.*
 
-[![Build Status](https://travis-ci.org/smarie/python-yamlable.svg?branch=master)](https://travis-ci.org/smarie/python-yamlable) [![Tests Status](https://smarie.github.io/python-yamlable/junit/junit-badge.svg?dummy=8484744)](https://smarie.github.io/python-yamlable/junit/report.html) [![codecov](https://codecov.io/gh/smarie/python-yamlable/branch/master/graph/badge.svg)](https://codecov.io/gh/smarie/python-yamlable) [![Documentation](https://img.shields.io/badge/docs-latest-blue.svg)](https://smarie.github.io/python-yamlable/) [![PyPI](https://img.shields.io/badge/PyPI-yamlable-blue.svg)](https://pypi.python.org/pypi/yamlable/)
+[![Python versions](https://img.shields.io/pypi/pyversions/yamlable.svg)](https://pypi.python.org/pypi/yamlable/) [![Build Status](https://travis-ci.org/smarie/python-yamlable.svg?branch=master)](https://travis-ci.org/smarie/python-yamlable) [![Tests Status](https://smarie.github.io/python-yamlable/junit/junit-badge.svg?dummy=8484744)](https://smarie.github.io/python-yamlable/junit/report.html) [![codecov](https://codecov.io/gh/smarie/python-yamlable/branch/master/graph/badge.svg)](https://codecov.io/gh/smarie/python-yamlable)
+
+[![Documentation](https://img.shields.io/badge/doc-latest-blue.svg)](https://smarie.github.io/python-yamlable/) [![PyPI](https://img.shields.io/pypi/v/yamlable.svg)](https://pypi.python.org/pypi/yamlable/) [![Downloads](https://pepy.tech/badge/yamlable)](https://pepy.tech/project/yamlable) [![Downloads per week](https://pepy.tech/badge/yamlable/week)](https://pepy.tech/project/yamlable) [![GitHub stars](https://img.shields.io/github/stars/smarie/python-yamlable.svg)](https://github.com/smarie/python-yamlable/stargazers)
 
 **This is the readme for developers.** The documentation for users is available here: [https://smarie.github.io/python-yamlable/](https://smarie.github.io/python-yamlable/)
 
@@ -34,9 +36,7 @@ This project uses `setuptools_scm` to synchronise the version number. Therefore
 python setup.py egg_info bdist_wheel rotate -m.whl -k3
 ```
 
-You need to [generate code](##building-from-sources--notes-on-this-projects-design-principles) before packaging.
-
-You also may need to install requirements for setup beforehand, using 
+You may need to install requirements for setup beforehand, using 
 
 ```bash
 pip install -r ci_tools/requirements-setup.txt
@@ -47,7 +47,7 @@ pip install -r ci_tools/requirements-setup.txt
 This project uses `mkdocs` to generate its documentation page. Therefore building a local copy of the doc page may be done using:
 
 ```bash
-mkdocs build
+mkdocs build -f docs/mkdocs.yml
 ```
 
 You may need to install requirements for doc beforehand, using 
@@ -74,3 +74,20 @@ This project is now automatically deployed to PyPI when a tag is created. Anyway
 twine upload dist/* -r pypitest
 twine upload dist/*
 ```
+
+### Merging pull requests with edits - memo
+
+Ax explained in github ('get commandline instructions'):
+
+```bash
+git checkout -b <git_name>-<feature_branch> master
+git pull https://github.com/<git_name>/python-yamlable.git <feature_branch> --no-commit --ff-only
+```
+
+if the second step does not work, do a normal auto-merge (do not use **rebase**!):
+
+```bash
+git pull https://github.com/<git_name>/python-yamlable.git <feature_branch> --no-commit
+```
+
+Finally review the changes, possibly perform some modifications, and commit.

ci_tools/generate-junit-badge.py

@@ -1,28 +1,70 @@
+import sys
+try:
+    # python 3
+    from urllib.parse import quote_plus
+except ImportError:
+    # python 2
+    from urllib import quote_plus
+
 import requests
 import shutil
 from os import makedirs, path
 import xunitparser
 
 
-def download_badge(junit_xml: str='reports/junit/junit.xml', dest_folder: str='reports/junit'):
+class TestStats(object):
+    def __init__(self, success_percentage, success, runned, skipped):
+        self.success_percentage = success_percentage
+        self.success = success
+        self.runned = runned
+        self.skipped = skipped
 
-    makedirs(dest_folder, exist_ok=True)
 
-    # read the junit test file
+def get_test_stats(junit_xml='reports/junit/junit.xml'  # type: str
+                   ):
+    # type: (...) -> TestStats
+    """
+    read the junit test file and extract the success percentage
+    :param junit_xml: the junit xml file path
+    :return: the success percentage (an int)
+    """
     ts, tr = xunitparser.parse(open(junit_xml))
-    runned = tr.testsRun
+    skipped = len(tr.skipped)
+    runned = tr.testsRun - skipped
     failed = len(tr.failures)
+    success = runned - failed
+
+    success_percentage = round(success * 100 / runned)
+
+    return TestStats(success_percentage, success, runned, skipped)
+
+
+def download_badge(test_stats,                  # type: TestStats
+                   dest_folder='reports/junit'  # type: str
+                   ):
+    """
+    Downloads the badge corresponding to the provided success percentage, from https://img.shields.io.
 
-    success_percentage = round((runned - failed) * 100 / runned)
-    if success_percentage < 50:
+    :param test_stats:
+    :param dest_folder:
+    :return:
+    """
+    if not path.exists(dest_folder):
+        makedirs(dest_folder)  # , exist_ok=True) not python 2 compliant
+
+    if test_stats.success_percentage < 50:
         color = 'red'
-    elif success_percentage < 75:
+    elif test_stats.success_percentage < 75:
         color = 'orange'
-    elif success_percentage < 90:
+    elif test_stats.success_percentage < 90:
         color = 'green'
     else:
         color = 'brightgreen'
-    url = 'https://img.shields.io/badge/tests-' + str(success_percentage) + '%25-' + color + '.svg'
+
+    left_txt = "tests"
+    # right_txt = "%s%%" % test_stats.success_percentage
+    right_txt = "%s/%s" % (test_stats.success, test_stats.runned)
+    url = 'https://img.shields.io/badge/%s-%s-%s.svg' % (left_txt, quote_plus(right_txt), color)
 
     dest_file = path.join(dest_folder, 'junit-badge.svg')
 
@@ -35,5 +77,19 @@ def download_badge(junit_xml: str='reports/junit/junit.xml', dest_folder: str='r
 
 
 if __name__ == "__main__":
-    # execute only if run as a script
-    download_badge()
+    # Execute only if run as a script.
+    # Check the arguments
+    assert len(sys.argv[1:]) == 1, "a single mandatory argument is required: <threshold>"
+    threshold = float(sys.argv[1])
+
+    # First retrieve the success percentage from the junit xml
+    test_stats = get_test_stats()
+
+    # Validate against the threshold
+    print("Success percentage is %s%%. Checking that it is >= %s" % (test_stats.success_percentage, threshold))
+    if test_stats.success_percentage < threshold:
+        raise Exception("Success percentage %s%% is strictly lower than required threshold %s%%"
+                        "" % (test_stats.success_percentage, threshold))
+
+    # Download the badge
+    download_badge(test_stats)