accept patch version too and run regression tests

Author: onurctirtir

.github/workflows/test-package-installation.yml

@@ -2,6 +2,11 @@ name: Test Latest Citus Package Installation on Supported PostgreSQL Versions
 
 on:
   workflow_dispatch:
+    inputs:
+      citus_version:
+        description: "Citus full version (e.g., 14.0.0)"
+        required: true
+        type: string
 
 env:
   # Branch whose build-package.yml and postgres-matrix.yml are the source of truth
@@ -13,7 +18,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.build_matrix.outputs.matrix }}
-      citus_version: ${{ steps.resolve_versions.outputs.citus_version }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -35,24 +39,36 @@ jobs:
           echo "Discovered platforms: ${platforms}"
           echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
 
-      - name: Resolve Citus & PG versions from postgres-matrix.yml
+      - name: Resolve PG versions from postgres-matrix.yml
         id: resolve_versions
         run: |
           git show "origin/${SOURCE_BRANCH}:postgres-matrix.yml" > /tmp/postgres-matrix.yml
 
-          # Latest entry = last element in version_matrix
-          citus_version=$(yq '.version_matrix[-1] | keys | .[0]' /tmp/postgres-matrix.yml)
+          citus_version="${{ inputs.citus_version }}"
+
+          # Validate x.y.z format
+          if ! [[ "$citus_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::citus_version must be in x.y.z format (e.g., 14.0.0), got: ${citus_version}"
+            exit 1
+          fi
+
+          # Extract major.minor for postgres-matrix.yml lookup
+          citus_major_minor="${citus_version%.*}"
 
           pg_versions=$(
             yq -o=json \
-              ".version_matrix[-1].\"${citus_version}\".postgres_versions" \
+              ".version_matrix[] | select(has(\"${citus_major_minor}\")).\"${citus_major_minor}\".postgres_versions" \
               /tmp/postgres-matrix.yml \
             | jq -c '[.[] | tostring]'
           )
 
-          echo "Latest Citus version: ${citus_version}"
+          if [[ -z "$pg_versions" || "$pg_versions" == "null" ]]; then
+            echo "::error::No entry found for Citus ${citus_major_minor} in postgres-matrix.yml"
+            exit 1
+          fi
+
+          echo "Citus version: ${citus_version} (major.minor: ${citus_major_minor})"
           echo "Supported PG versions: ${pg_versions}"
-          echo "citus_version=${citus_version}" >> "$GITHUB_OUTPUT"
           echo "pg_versions=${pg_versions}" >> "$GITHUB_OUTPUT"
 
       - name: Build matrix
@@ -66,7 +82,7 @@ jobs:
           echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
 
   test_package:
-    name: "PG${{ matrix.pg_version }}/Citus${{ needs.discover.outputs.citus_version }} - ${{ matrix.platform }}"
+    name: "PG${{ matrix.pg_version }}/Citus${{ inputs.citus_version }} - ${{ matrix.platform }}"
     needs: discover
     runs-on: ubuntu-latest
     strategy:
@@ -86,4 +102,4 @@ jobs:
           python3 test_package_installation/run_test.py \
             "${{ matrix.platform }}" \
             "${{ matrix.pg_version }}" \
-            "${{ needs.discover.outputs.citus_version }}"
+            "${{ inputs.citus_version }}"

test_package_installation/README.md

@@ -2,29 +2,33 @@
 
 Verifies that Citus packages can be installed from the package repository on all supported platforms.
 
-Each test spins up a Docker container for the target OS, installs the PostgreSQL and Citus packages,
-sets up a cluster, and runs very basic tests.
+Each test:
+* spins up a Docker container for the target OS
+* installs the PostgreSQL and Citus packages
+* sets up a cluster and runs very basic tests
+* runs `make check-multi` regression tests from the Citus repository against the installed Citus version
 
 ## Running locally
 
 Requires **Python 3.10+** and **Docker**. No additional Python packages needed.
 
 ```bash
-python3 test_package_installation/run_test.py <platform> <pg_major_version> <citus_major_minor_version>
+python3 test_package_installation/run_test.py <platform> <pg_major_version> <citus_full_version>
 ```
 
 Examples:
 
 ```bash
-python3 test_package_installation/run_test.py ubuntu/noble 18 14.0
-python3 test_package_installation/run_test.py el/9 17 14.0
-python3 test_package_installation/run_test.py debian/bookworm 16 14.0
+python3 test_package_installation/run_test.py ubuntu/noble 18 14.0.0
+python3 test_package_installation/run_test.py el/9 17 14.0.0
+python3 test_package_installation/run_test.py debian/bookworm 16 14.0.0
 ```
 
 ## Running via GitHub Actions
 
 You can manually trigger the workflow using [this GitHub Actions tab](https://github.com/citusdata/packaging/actions/workflows/test-package-installation.yml).
 
-It automatically discovers the supported platforms, latest Citus version that we published
-packages for, and compatible PostgreSQL versions from the `all-citus` branch, using
-[.github/workflows/build-package.yml](https://github.com/citusdata/packaging/blob/all-citus/.github/workflows/build-package.yml) and [postgres-matrix.yml](https://github.com/citusdata/packaging/blob/all-citus/postgres-matrix.yml) files.
+It asks for the Citus version to test, and then it automatically discovers the supported platforms and
+compatible PostgreSQL versions from the `all-citus` branch, using
+[.github/workflows/build-package.yml](https://github.com/citusdata/packaging/blob/all-citus/.github/workflows/build-package.yml)
+and [postgres-matrix.yml](https://github.com/citusdata/packaging/blob/all-citus/postgres-matrix.yml) files.

test_package_installation/common/run_multi_schedule.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Runs "make check-multi" (multi_schedule).
+#
+# Expects the Citus source to be cloned at /citus and pg_regress
+# to be available in the PostgreSQL binary path.
+#
+# Usage: run_multi_schedule.sh <pg_binary_path>
+# Example: run_multi_schedule.sh /usr/lib/postgresql/18/bin
+
+set -euo pipefail
+
+if [ -z "$1" ]; then
+    echo "Error: some arguments are missing." >&2
+    echo "Usage: $0 <pg_binary_path>" >&2
+    exit 1
+fi
+
+PG_BIN="$1"
+
+## suppress some of the regression test failures
+
+# On rpm based distros, Postgres package might be linked against a newer ICU
+# than the one used to generate expected test output. To avoid test failures
+# due to this, patch the expected output to match the newer ICU version.
+# Without this, today pg18.sql fails.
+echo 's/und-u-kc-true-ks-level1/und-u-kc-ks-level1/g' >> /citus/src/test/regress/bin/normalize.sed
+
+## run "make check-multi"
+
+export PATH="${PG_BIN}:${PATH}"
+
+cd /citus
+
+# we don't need lz4 and zstd as none of the tests in multi_schedule depend on them
+./configure PG_CONFIG="${PG_BIN}/pg_config" --without-libcurl --without-lz4 --without-zstd
+
+chown -R postgres:postgres /citus
+
+echo "Running make check-multi ..."
+
+if ! su - postgres -c "make check-multi -C /citus/src/test/regress"; then
+    echo "make check-multi failed. Printing regression.diffs ..."
+    cat /citus/src/test/regress/regression.diffs
+    exit 1
+fi
+
+echo "make check-multi completed successfully."

…e_installation/setup_cluster_and_test.sh …ckage_installation/common/test_basics.shtest_package_installation/setup_cluster_and_test.sh renamed to test_package_installation/common/test_basics.sh test_package_installation/setup_cluster_and_test.sh renamed to test_package_installation/common/test_basics.sh

@@ -3,31 +3,30 @@
 # Sets up a multi-node Citus cluster, creates a distributed table,
 # rebalances shards, and verifies even distribution.
 #
-# Called by test_deb.sh and test_rpm.sh after package installation.
-#
 # Note: need to run this as root
 #
-# Usage: setup_cluster_and_test.sh <pg_binary_path>
-# Example: setup_cluster_and_test.sh /usr/lib/postgresql/17/bin
+# Usage: test_basics.sh <pg_major_version> <citus_full_version> <pg_binary_path>
+# Example: test_basics.sh 18 14.0.0 /usr/lib/postgresql/17/bin
 
 set -euo pipefail
 
-if [ -z "$1" ]; then
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     echo "Error: some arguments are missing." >&2
-    echo "Usage: $0 <pg_binary_path>" >&2
+    echo "Usage: $0 <pg_major_version> <citus_full_version> <pg_binary_path>" >&2
     exit 1
 fi
 
-PG_BIN="$1"
+PG_MAJOR_VERSION="$1"
+CITUS_FULL_VERSION="$2"
+PG_BIN="$3"
 
 ## constants and helpers
 
 NODE_COUNT=3
 SHARDS_PER_NODE=5
 FIRST_NODE_PORT=9700
 
-run_psql_cmd()
-{
+run_psql_cmd() {
     if [ -z "$1" ] || [ -z "$2" ]; then
         echo "Error: some arguments are missing." >&2
         echo "Usage: run_psql_cmd <port_number> <sql_command>" >&2
@@ -62,6 +61,26 @@ for i in $(seq 0 $((NODE_COUNT - 1))); do
     run_psql_cmd "$port_number" "CREATE EXTENSION citus;"
 done
 
+## verify PG version
+
+pg_version=$(run_psql_cmd "$FIRST_NODE_PORT" "SELECT version();")
+if [[ "$pg_version" != "PostgreSQL ${PG_MAJOR_VERSION}."* ]]; then
+    echo "Error: PostgreSQL version verification failed. Expected version to start with 'PostgreSQL ${PG_MAJOR_VERSION}.', got: $pg_version" >&2
+    exit 1
+else
+    echo "Verified PostgreSQL version: $pg_version"
+fi
+
+## verify Citus extension version
+
+citus_version=$(run_psql_cmd "$FIRST_NODE_PORT" "SHOW citus.version;")
+if [[ "$citus_version" != "${CITUS_FULL_VERSION}"* ]]; then
+    echo "Error: Citus version verification failed. Expected version to start with '${CITUS_FULL_VERSION}', got: $citus_version" >&2
+    exit 1
+else
+    echo "Verified Citus version: $citus_version"
+fi
+
 ## create a distributed table
 
 echo "Creating a distributed table ..."

test_package_installation/test_deb.sh …kage_installation/deb/install_package.shtest_package_installation/test_deb.sh renamed to test_package_installation/deb/install_package.sh test_package_installation/test_deb.sh renamed to test_package_installation/deb/install_package.sh

@@ -5,27 +5,28 @@
 #
 # Note: need to run this as root
 #
-# Usage: test_deb.sh <pg_major_version> <citus_major_minor_version>
-# Example: test_deb.sh 17 14.0
+# Usage: install_package.sh <pg_major_version> <citus_full_version>
+# Example: install_package.sh 18 14.0.0
 
 set -euo pipefail
 
 if [ -z "$1" ] || [ -z "$2" ]; then
     echo "Error: some arguments are missing." >&2
-    echo "Usage: $0 <pg_major_version> <citus_major_minor_version>" >&2
+    echo "Usage: $0 <pg_major_version> <citus_full_version>" >&2
     exit 1
 fi
 
 PG_MAJOR_VERSION="$1"
-CITUS_MAJOR_MINOR_VERSION="$2"
+CITUS_FULL_VERSION="$2"
 
 export DEBIAN_FRONTEND=noninteractive
 
+apt-get -y update
+
 ## install postgres
 
 echo "Installing PostgreSQL ..."
 
-apt-get -y update
 apt-get -y install curl ca-certificates
 
 install -d /usr/share/postgresql-common/pgdg
@@ -42,8 +43,8 @@ echo "Installing Citus ..."
 
 curl --fail https://install.citusdata.com/community/deb.sh > add-citus-repo.sh
 bash add-citus-repo.sh
-apt-get -y install postgresql-${PG_MAJOR_VERSION}-citus-${CITUS_MAJOR_MINOR_VERSION}
 
-## setup a citus cluster and run verification tests
+citus_major_minor_version="${CITUS_FULL_VERSION%.*}"
+apt-get -y install postgresql-${PG_MAJOR_VERSION}-citus-${citus_major_minor_version}=${CITUS_FULL_VERSION}.citus-1
 
-/setup_cluster_and_test.sh "/usr/lib/postgresql/${PG_MAJOR_VERSION}/bin"
+echo "PostgreSQL and Citus packages installed successfully."

test_package_installation/deb/install_regression_test_deps.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Installs dependencies needed to run Citus regression tests.
+#
+# Note: need to run this as root
+#
+# Usage: install_regression_test_deps.sh <pg_major_version> <citus_full_version>
+# Example: install_regression_test_deps.sh 18 14.0.0
+
+set -euo pipefail
+
+if [ -z "$1" ] || [ -z "$2" ]; then
+    echo "Error: some arguments are missing." >&2
+    echo "Usage: $0 <pg_major_version> <citus_full_version>" >&2
+    exit 1
+fi
+
+PG_MAJOR_VERSION="$1"
+CITUS_FULL_VERSION="$2"
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get -y update
+
+## install pg_regress
+
+echo "Installing pg_regress via postgresql-server-dev-${PG_MAJOR_VERSION} ..."
+
+apt-get -y install postgresql-server-dev-${PG_MAJOR_VERSION}
+
+## clone citus repository
+
+echo "Cloning Citus repository (tag v${CITUS_FULL_VERSION}) ..."
+
+CITUS_TAG="v${CITUS_FULL_VERSION}"
+CITUS_REPO_DIR="/citus"
+
+apt-get -y install git build-essential
+git clone --depth 1 --branch "$CITUS_TAG" https://github.com/citusdata/citus "$CITUS_REPO_DIR"
+
+echo "Regression test dependencies installed successfully."

test_package_installation/rpm/install_package.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Installs PostgreSQL and Citus packages on EL/OL (RPM-based distros), then
+# runs the cluster setup and verification test.
+#
+# Note: need to run this as root
+#
+# Usage: install_package.sh <pg_major_version> <citus_full_version> <os_version>
+# Example: install_package.sh 18 14.0.0 8
+
+set -euo pipefail
+
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "Error: some arguments are missing." >&2
+    echo "Usage: $0 <pg_major_version> <citus_full_version> <os_version>" >&2
+    exit 1
+fi
+
+PG_MAJOR_VERSION="$1"
+CITUS_FULL_VERSION="$2"
+OS_VERSION="$3"
+
+yum -y update
+
+## install postgres
+
+echo "Installing PostgreSQL ..."
+
+yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-${OS_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm
+
+# disable the built-in PostgreSQL module
+yum -qy module disable postgresql
+
+yum install -y postgresql${PG_MAJOR_VERSION}-server
+
+## install citus
+
+echo "Installing Citus ..."
+
+curl --fail https://install.citusdata.com/community/rpm.sh > add-citus-repo.sh
+bash add-citus-repo.sh
+
+citus_major_minor_version="${CITUS_FULL_VERSION%.*}"
+citus_major_minor_version_without_dot="${citus_major_minor_version//./}"
+
+# TODO: use --nogpgcheck for now
+yum install -y --nogpgcheck citus${citus_major_minor_version_without_dot}_${PG_MAJOR_VERSION}-${CITUS_FULL_VERSION}.citus-1.el${OS_VERSION}
+
+echo "PostgreSQL and Citus packages installed successfully."