Merge branch 'develop' into issue-2783-bernoulli-cdf-stable

Author: andrjohns

.git-blame-ignore-revs

@@ -25,4 +25,4 @@ If this is a **feature request**, show what you expect to happen if the feature
 
 
 #### Current Version:
-v4.5.0
+v4.6.2

.github/ISSUE_TEMPLATE.md

@@ -16,21 +16,16 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   windows:
-    permissions:
-      actions: write  # for n1hility/cancel-previous-runs to create & stop workflow runs
-      contents: read  # for actions/checkout to fetch code
     name: Windows
     runs-on: windows-latest
 
     steps:
-    - uses: n1hility/cancel-previous-runs@v2
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: c-cpp.yml
-      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'"
-
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
@@ -53,39 +48,21 @@ jobs:
       run: make -j2 test-headers
 
   opencl:
-    permissions:
-      actions: write  # for n1hility/cancel-previous-runs to create & stop workflow runs
-      contents: read  # for actions/checkout to fetch code
     name: OpenCL
     runs-on: ubuntu-latest
 
     steps:
-    - uses: n1hility/cancel-previous-runs@v2
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: c-cpp.yml
-      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'"
-
     - uses: actions/checkout@v3
 
     - name: Run header tests
       run: |
         echo "STAN_OPENCL=true" > make/local
         make -j2 test-headers
   no_range_checks:
-    permissions:
-      actions: write  # for n1hility/cancel-previous-runs to create & stop workflow runs
-      contents: read  # for actions/checkout to fetch code
     name: NoRange
     runs-on: ubuntu-latest
 
     steps:
-    - uses: n1hility/cancel-previous-runs@v2
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: c-cpp.yml
-      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'"
-
     - uses: actions/checkout@v3
 
     - name: Run header tests

.github/workflows/header_checks.yml

@@ -14,21 +14,16 @@ on:
       - 'RELEASE-NOTES.txt'
 permissions:
   contents: read
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 
 jobs:
   prim-rev:
-    permissions:
-      actions: write  # for n1hility/cancel-previous-runs to create & stop workflow runs
-      contents: read  # for actions/checkout to fetch code
     name: prim and rev tests
     runs-on: windows-latest
 
     steps:
-    - uses: n1hility/cancel-previous-runs@v2
-      with:
-        token: ${{ secrets.GITHUB_TOKEN }}
-      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'"
-
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
@@ -73,6 +68,7 @@ jobs:
       with:
         name: gtest_outputs_xml
         path: '**/*_test.xml'
+
   fwd-non-fun-mix:
     name: fwd tests and non-fun mix tests
     runs-on: windows-latest
@@ -126,6 +122,7 @@ jobs:
       with:
         name: gtest_outputs_xml
         path: '**/*_test.xml'
+
   mix-fun-1:
     name: mix/fun tests 1
     runs-on: windows-latest
@@ -174,6 +171,7 @@ jobs:
       with:
         name: gtest_outputs_xml
         path: '**/*_test.xml'
+        
   mix-fun-2:
     name: mix/fun tests 2
     runs-on: windows-latest

.github/workflows/main.yml

@@ -14,7 +14,7 @@ def runTests(String testPath, boolean jumbo = false) {
 }
 
 def skipRemainingStages = false
-
+def changedDistributionTests = []
 def utils = new org.stan.Utils()
 
 def isBranch(String b) { env.BRANCH_NAME == b }
@@ -38,11 +38,12 @@ pipeline {
         booleanParam(defaultValue: false, name: 'withRowVector', description: 'Run additional distribution tests on RowVectors (takes 5x as long)')
         booleanParam(defaultValue: false, name: 'disableJumbo', description: 'Disable Jumbo tests. This takes longer and should only be used for debugging if it is believed that the jumbo tests are causing failures.')
         booleanParam(defaultValue: false, name: 'optimizeUnitTests', description: 'Use O=3 for unit tests (takex ~3x as long)')
+        booleanParam(defaultValue: false, name: 'runAllDistributions', description: 'Run all distribution tests, even ones which are unchanged compared to develop')
     }
     options {
         skipDefaultCheckout()
         preserveStashes(buildCount: 7)
-	parallelsAlwaysFailFast()
+        parallelsAlwaysFailFast()
     }
     environment {
         STAN_NUM_THREADS = 4
@@ -57,6 +58,10 @@ pipeline {
         OPENCL_PLATFORM_ID_CPU = 0
         OPENCL_PLATFORM_ID_GPU = 0
         PARALLEL = 4
+        GIT_AUTHOR_NAME = 'Stan Jenkins'
+        GIT_AUTHOR_EMAIL = 'mc.stanislaw@gmail.com'
+        GIT_COMMITTER_NAME = 'Stan Jenkins'
+        GIT_COMMITTER_EMAIL = 'mc.stanislaw@gmail.com'
     }
     stages {
 
@@ -85,14 +90,12 @@ pipeline {
                     usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) {
                     sh """#!/bin/bash
                         set -x
-                        git config user.email "mc.stanislaw@gmail.com"
-                        git config user.name "Stan Jenkins"
                         git checkout -b ${branchName()}
                         clang-format --version
                         find stan test -name '*.hpp' -o -name '*.cpp' | xargs -n20 -P${PARALLEL} clang-format -i
                         if [[ `git diff` != "" ]]; then
                             git add stan test
-                            git commit --author='Stan BuildBot <mc.stanislaw@gmail.com>' -m "[Jenkins] auto-formatting by `clang-format --version`"
+                            git commit -m "[Jenkins] auto-formatting by `clang-format --version`"
                             git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${fork()}/math.git ${branchName()}
                             echo "Exiting build because clang-format found changes."
                             echo "Those changes are now found on stan-dev/math under branch ${branchName()}"
@@ -121,7 +124,7 @@ pipeline {
                     }
                 }
             }
-         }
+        }
 
         stage('Linting & Doc checks') {
             agent {
@@ -175,27 +178,65 @@ pipeline {
 
                 }
             }
+            post { always { retry(3) { deleteDir() } } }
         }
 
-        stage('Headers check') {
-            agent {
-                docker {
-                    image 'stanorg/ci:gpu-cpp17'
-                    label 'linux'
-                }
-            }
+        stage('Quick tests') {
             when {
                 expression {
                     !skipRemainingStages
                 }
             }
-            steps {
-                unstash 'MathSetup'
-                sh "echo CXX=${CLANG_CXX} -Werror > make/local"
-                sh "echo O=0 >> make/local"
-                sh "make -j${PARALLEL} test-headers"
+            failFast true
+            parallel {
+                stage('Headers check') {
+                    when {
+                        expression {
+                            !skipRemainingStages
+                        }
+                    }
+                    agent {
+                        docker {
+                            image 'stanorg/ci:gpu-cpp17'
+                            label 'linux'
+                        }
+                    }
+
+                    steps {
+                        unstash 'MathSetup'
+                        sh "echo CXX=${CLANG_CXX} -Werror > make/local"
+                        sh "make -j${PARALLEL} test-headers"
+                    }
+                    post { always { deleteDir() } }
+                }
+                stage('Run changed unit tests') {
+                    agent {
+                        docker {
+                            image 'stanorg/ci:gpu-cpp17'
+                            label 'linux'
+                            args '--cap-add SYS_PTRACE'
+                        }
+                    }
+                    when {
+                        allOf {
+                            expression {
+                                env.BRANCH_NAME ==~ /PR-\d+/
+                            }
+                            expression {
+                                !skipRemainingStages
+                            }
+                        }
+                    }
+                    steps {
+                        retry(3) { checkout scm }
+
+                        sh "echo CXXFLAGS += -fsanitize=address >> make/local"
+                        sh "./runTests.py -j${PARALLEL} --changed --debug"
+
+                    }
+                    post { always { retry(3) { deleteDir() } } }
+                }
             }
-            post { always { deleteDir() } }
         }
 
         stage('Full Unit Tests') {
@@ -406,35 +447,66 @@ pipeline {
             }
         }
 
-        stage ('Distribution tests') {
+        stage ('Discover changed distribution tests') {
             when {
                 expression {
                     !skipRemainingStages
                 }
             }
+            agent {
+                docker {
+                    image 'stanorg/ci:gpu-cpp17'
+                    label 'linux'
+                }
+            }
+            steps {
+                script {
+                    retry(3) { checkout scm }
+                    if (params.runAllDistributions || isBranch('develop') || isBranch('master')) {
+                        changedDistributionTests = sh(script:"python3 test/prob/getDependencies.py --pretend-all", returnStdout:true).trim().readLines()
+                    } else {
+                        changedDistributionTests = sh(script:"python3 test/prob/getDependencies.py", returnStdout:true).trim().readLines()
+                    }
+                }
+            }
+        }
+
+        stage ('Distribution tests') {
+            when {
+                allOf {
+                    expression {
+                        !skipRemainingStages
+                    }
+                    expression {
+                        !changedDistributionTests.isEmpty()
+                    }
+                }
+            }
             agent { label 'linux && docker' }
             steps {
                 script {
-                    unstash 'MathSetup'
                     def tests = [:]
-                    def files = sh(script:"find test/prob/* -type d", returnStdout:true).trim().split('\n')
-                    for (f in files.toList().collate(8)) {
+                    for (f in changedDistributionTests.collate(24)) {
                         def names = f.join(" ")
                         tests["Distribution Tests: ${names}"] = { node ("linux && docker") {
+                            deleteDir()
                             docker.image('stanorg/ci:gpu-cpp17').inside {
-                                unstash 'MathSetup'
-                                sh """
-                                    echo CXX=${CLANG_CXX} > make/local
-                                    echo O=0 >> make/local
-                                    echo N_TESTS=${N_TESTS} >> make/local
-                                    """
-                                script {
-                                    if (params.withRowVector || isBranch('develop') || isBranch('master')) {
-                                        sh "echo CXXFLAGS+=-DSTAN_TEST_ROW_VECTORS >> make/local"
-                                        sh "echo CXXFLAGS+=-DSTAN_PROB_TEST_ALL >> make/local"
+                                catchError {
+                                    unstash 'MathSetup'
+                                    sh """
+                                        echo CXX=${CLANG_CXX} > make/local
+                                        echo O=0 >> make/local
+                                        echo N_TESTS=${N_TESTS} >> make/local
+                                        """
+                                    script {
+                                        if (params.withRowVector || isBranch('develop') || isBranch('master')) {
+                                            sh "echo CXXFLAGS+=-DSTAN_TEST_ROW_VECTORS >> make/local"
+                                            sh "echo CXXFLAGS+=-DSTAN_PROB_TEST_ALL >> make/local"
+                                        }
                                     }
+                                    sh "./runTests.py -j${PARALLEL} ${names}"
                                 }
-                                sh "./runTests.py -j${PARALLEL} ${names}"
+                                deleteDir()
                             }
                         } }
                     }

Jenkinsfile

@@ -1,28 +1,28 @@
-<div><a href="https://zenodo.org/badge/latestdoi/38388440"><img src="https://zenodo.org/badge/38388440.svg"/></a></div>
-
-The <b>Stan Math Library</b> is a C++, reverse-mode automatic
-differentiation library designed to be usable, extensive and
-extensible, efficient, scalable, stable, portable, and redistributable
-in order to facilitate the construction and utilization of algorithms
-that utilize derivatives.
-
-
-Documentation, Installation, and Examples
---------------
-
-All of Stan math's documentation is hosted on our website below. Please do not
-reference articles in the wiki as they are outdated and not maintained.
-
-[mc-stan.org/math](https://mc-stan.org/math/)
-
-
-Licensing
----------
-The Stan Math Library is licensed under the [new BSD
-license](https://github.com/stan-dev/math/blob/develop/LICENSE%2Emd).
-
-The Stan Math Library depends on the Intel TBB library which is
-licensed under the Apache 2.0 license. This dependency implies an
-additional restriction as compared to the new BSD license alone. The
-Apache 2.0 license is incompatible with GPL-2 licensed code if
-distributed as a unitary binary. You may refer to the Licensing page on the [Stan wiki](https://github.com/stan-dev/stan/wiki/Stan-Licensing).
+<div><a href="https://zenodo.org/badge/latestdoi/38388440"><img src="https://zenodo.org/badge/38388440.svg"/></a></div>
+
+The <b>Stan Math Library</b> is a C++, reverse-mode automatic
+differentiation library designed to be usable, extensive and
+extensible, efficient, scalable, stable, portable, and redistributable
+in order to facilitate the construction and utilization of algorithms
+that utilize derivatives.
+
+
+Documentation, Installation, and Examples
+--------------
+
+All of Stan math's documentation is hosted on our website below. Please do not
+reference articles in the wiki as they are outdated and not maintained.
+
+[mc-stan.org/math](https://mc-stan.org/math/)
+
+
+Licensing
+---------
+The Stan Math Library is licensed under the [new BSD
+license](https://github.com/stan-dev/math/blob/develop/LICENSE%2Emd).
+
+The Stan Math Library depends on the Intel TBB library which is
+licensed under the Apache 2.0 license. This dependency implies an
+additional restriction as compared to the new BSD license alone. The
+Apache 2.0 license is incompatible with GPL-2 licensed code if
+distributed as a unitary binary. You may refer to the Licensing page on the [Stan wiki](https://github.com/stan-dev/stan/wiki/Stan-Licensing).