deployment workflow enhancement to test correct release

orbeckst · orbeckst · commit 0d8c4d6056af · 2026-04-02T18:14:06.000-07:00
- pass version number from build to installing from pypi
- new action/wait-for-pypi-version that waits until the correct package is on pypi or
  testpypi so that we can download and test properly
- testing after pypi deployment fix
  - ensure that testing does not see the checked out sources because then it will
    be missing authors.py and fail the author test
  - check out repo for the wait-for-pypi action under source
- based on Becksteinlab/multibind/.github/workflows (MIT license)
diff --git a/.github/actions/wait-for-pypi-version/action.yaml b/.github/actions/wait-for-pypi-version/action.yaml
@@ -0,0 +1,93 @@
+name: 'Wait for PyPI version'
+description: 'Wait for a specific package version to become available on PyPI or TestPyPI'
+inputs:
+  repository:
+    description: 'PyPI repository type: "pypi" or "testpypi"'
+    required: true
+  package:
+    description: 'Package name'
+    required: true
+  version:
+    description: 'Package version to wait for'
+    required: true
+  max_attempts:
+    description: 'Maximum number of retry attempts'
+    required: false
+    default: '30'
+  wait_seconds:
+    description: 'Seconds to wait between attempts'
+    required: false
+    default: '10'
+
+runs:
+  using: composite
+  steps:
+    - name: Install requests
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install requests
+
+    - name: Wait for version to be available
+      shell: python
+      env:
+        REPOSITORY: ${{ inputs.repository }}
+        PACKAGE: ${{ inputs.package }}
+        VERSION: ${{ inputs.version }}
+        MAX_ATTEMPTS: ${{ inputs.max_attempts }}
+        WAIT_SECONDS: ${{ inputs.wait_seconds }}
+      run: |
+        import os
+        import sys
+        import time
+
+        import requests
+
+        repository = os.environ["REPOSITORY"].strip().lower()
+        package = os.environ["PACKAGE"]
+        version = os.environ["VERSION"]
+        max_attempts = int(os.environ.get("MAX_ATTEMPTS", "30"))
+        wait_seconds = int(os.environ.get("WAIT_SECONDS", "10"))
+
+        if repository == "testpypi":
+            api_url = f"https://test.pypi.org/pypi/{package}/json"
+            repo_name = "TestPyPI"
+        elif repository == "pypi":
+            api_url = f"https://pypi.org/pypi/{package}/json"
+            repo_name = "PyPI"
+        else:
+            print(
+                f"ERROR: repository must be 'pypi' or 'testpypi', got {repository!r}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        for attempt in range(max_attempts):
+            try:
+                r = requests.get(api_url, timeout=10)
+                r.raise_for_status()
+                data = r.json()
+                versions = data.get("releases", {})
+                keys = list(versions.keys())
+                print("Available versions:", keys[-10:])  # Show last 10 versions
+                if version in versions:
+                    print(f"✓ Version {version} is available on {repo_name}")
+                    print(f"Version {version} is now available on {repo_name}")
+                    sys.exit(0)
+                print(f"✗ Version {version} is NOT available on {repo_name}")
+            except Exception as e:
+                print(f"Error checking version: {e}")
+
+            current = attempt + 1
+            print(
+                f"Attempt {current}/{max_attempts}: Version {version} not yet available "
+                f"on {repo_name}, waiting {wait_seconds} seconds..."
+            )
+            time.sleep(wait_seconds)
+
+        print(
+            f"ERROR: Version {version} did not become available on {repo_name} "
+            f"after {max_attempts} attempts",
+            file=sys.stderr,
+        )
+        sys.exit(1)
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -15,15 +15,19 @@ concurrency:
 defaults:
   run:
     shell: bash -l {0}
-  
 
 jobs:
   build:
     name: Build package
     runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.extract-version.outputs.version }}
     steps:
       - name: Checkout
         uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          fetch-tags: true
 
       - name: Set up Python
         uses: actions/setup-python@v6
@@ -36,12 +40,24 @@ jobs:
           pip install build twine
 
       - name: Build package (binary wheel and source distribution package)
-        run: |
-          python -m build
+        run: python -m build
 
       - name: Check package
+        run: twine check dist/*
+
+      - name: Extract package version
+        id: extract-version
         run: |
-          twine check dist/*
+          WHEEL_FILE=$(ls dist/*.whl)
+          VERSION=$(basename "$WHEEL_FILE" | sed -n 's/mdanalysisdata-\([^-]*\)-.*/\1/p')
+          if [ -z "$VERSION" ]; then
+            python -m pip install --upgrade pip
+            pip install "$WHEEL_FILE" --quiet
+            VERSION=$(python -c "import MDAnalysisData; print(MDAnalysisData.__version__)")
+            pip uninstall -y MDAnalysisData --quiet
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Extracted version: $VERSION"
 
       - name: Upload dist files
         uses: actions/upload-artifact@v7
@@ -70,13 +86,13 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           WHEEL_FILE=$(ls dist/*.whl)
-          pip install "$WHEEL_FILE"[test]
+          pip install "${WHEEL_FILE}[test]"
 
       - name: Test import
         run: |
           python -c "import MDAnalysisData; print(f'Package {MDAnalysisData.__version__} imported successfully')"
           
-      - name: Run basic tests
+      - name: Run tests (without data file downloads)
         run: |
           pytest --verbose -m "not online" --pyargs MDAnalysisData
 
@@ -135,7 +151,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-    needs: deploy-testpypi
+    needs: [build, deploy-testpypi]
     if: |
       github.repository == 'MDAnalysis/MDAnalysisData' &&
       (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/'))
@@ -145,16 +161,28 @@ jobs:
         with:
           python-version: "3.14"
 
+      - name: Checkout repository for actions
+        uses: actions/checkout@v6
+        with:
+          path: source    # put code under different path and do not cd to avoid interference with pytest invocation (missing authors.py)
+
+      - name: Wait for version to be available on TestPyPI
+        uses: ./source/.github/actions/wait-for-pypi-version
+        with:
+          repository: testpypi
+          package: MDAnalysisData
+          version: ${{ needs.build.outputs.version }}
+
       - name: Install from TestPyPI
         run: |
           python -m pip install --upgrade pip
-          pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ MDAnalysisData[test]
+          pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "MDAnalysisData[test]==${{ needs.build.outputs.version }}"
 
       - name: Test import
         run: |
           python -c "import MDAnalysisData; print(f'Package {MDAnalysisData.__version__} imported successfully from TestPyPi')"
 
-      - name: Run basic tests
+      - name: Run tests
         run: |
           pytest --verbose -m "not online" --pyargs MDAnalysisData
 
@@ -165,7 +193,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-    needs: deploy-pypi
+    needs: [build, deploy-pypi]
     if: |
       github.repository == 'MDAnalysis/MDAnalysisData' &&
       (github.event_name == 'release' && github.event.action == 'published')
@@ -175,15 +203,27 @@ jobs:
         with:
           python-version: "3.14"
 
+      - name: Checkout repository for actions
+        uses: actions/checkout@v6
+        with:
+          path: source    # put code under different path and do not cd to avoid interference with pytest invocation (missing authors.py)
+
+      - name: Wait for version to be available on PyPI
+        uses: ./source/.github/actions/wait-for-pypi-version
+        with:
+          repository: pypi
+          package: MDAnalysisData
+          version: ${{ needs.build.outputs.version }}
+
       - name: Install from PyPI
         run: |
           python -m pip install --upgrade pip
-          pip install MDAnalysisData[test]
+          pip install "MDAnalysisData[test]==${{ needs.build.outputs.version }}"
 
       - name: Test import
         run: |
           python -c "import MDAnalysisData; print(f'Package {MDAnalysisData.__version__} imported successfully from PyPi')"
 
-      - name: Run basic tests
+      - name: Run tests
         run: |
           pytest --verbose -m "not online" --pyargs MDAnalysisData
