Skip to content

Commit e8e85a1

Browse files
author
Gürkan İndibay
authored
Citus packages test infrastructure (#196)
1 parent 9d6d50f commit e8e85a1

18 files changed

Lines changed: 1079 additions & 17 deletions

File tree

‎.github/workflows/package-tests.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
name: Citus package tests
2+
3+
on:
4+
push:
5+
branches:
6+
- "**"
7+
8+
workflow_dispatch:
9+
inputs:
10+
prj_ver:
11+
description: "The version to be tested"
12+
required: true
13+
14+
jobs:
15+
metadata:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }}
19+
steps:
20+
- name: Checkout
21+
uses: actions/checkout@v2
22+
with:
23+
fetch-depth: 2
24+
- name: Install dependencies
25+
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
26+
- name: Install python requirements
27+
run: python -m pip install -r packaging_automation/requirements.txt
28+
- name: generate postgres
29+
id: generate-postgres
30+
run: |
31+
export PROJECT_VERSION="${{ github.event.inputs.name }}"
32+
[ -z ${PROJECT_VERSION} ] && export PROJECT_VERSION=10.2.1
33+
POSTGRES_VERSIONS=$(python -m packaging_automation.get_postgres_versions --prj_ver ${PROJECT_VERSION})
34+
echo "Postgres Version: ${POSTGRES_VERSIONS}"
35+
echo "::set-output name=pg_versions::${POSTGRES_VERSIONS}"
36+
test_execution:
37+
runs-on: ubuntu-latest
38+
needs: metadata
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
platform:
43+
- centos/8
44+
- centos/7
45+
- ol/7
46+
- debian/stretch
47+
- debian/buster
48+
- debian/bullseye
49+
- ubuntu/bionic
50+
- ubuntu/focal
51+
pg: ${{ fromJson(needs.metadata.outputs.pg_versions) }}
52+
env:
53+
PLATFORM: ${{ matrix.platform }}
54+
55+
steps:
56+
- name: Checkout repository
57+
uses: actions/checkout@v2
58+
59+
- name: Install dependencies
60+
run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
61+
62+
- name: Install python requirements
63+
run: python -m pip install -r packaging_automation/requirements.txt
64+
65+
- name: Citus package tests
66+
run: |
67+
export PROJECT_VERSION="${{ github.event.inputs.name }}"
68+
[ -z ${PROJECT_VERSION} ]&& export PROJECT_VERSION=10.2.1
69+
python -m packaging_automation.test_citus_package \
70+
--prj_ver "${PROJECT_VERSION}" \
71+
--os_release ${{ matrix.platform }} \
72+
--pg_major_version ${{ matrix.pg }}

‎packaging_automation/citus_package.py‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
get_gpg_fingerprints_by_name,
1818
get_supported_postgres_nightly_versions,
1919
get_supported_postgres_release_versions,
20+
platform_names,
2021
run_with_output, str_array_to_str,
22+
supported_platforms,
2123
transform_key_into_base64_str)
2224
from .packaging_warning_handler import validate_output
2325

@@ -26,23 +28,6 @@
2628
POSTGRES_VERSION_FILE = "supported-postgres"
2729
POSTGRES_MATRIX_FILE_NAME = "postgres-matrix.yml"
2830

29-
supported_platforms = {
30-
"debian": ["bullseye", "buster", "stretch", "jessie", "wheezy"],
31-
"el": ["8", "7", "6"],
32-
"ol": ["7", "8"],
33-
"ubuntu": ["focal", "bionic", "xenial", "trusty"]
34-
}
35-
36-
37-
def platform_names() -> List[str]:
38-
platforms = []
39-
for platform_os, platform_releases in supported_platforms.items():
40-
for platform_release in platform_releases:
41-
platforms.append(f"{platform_os}/{platform_release}")
42-
platforms.append("pgxn")
43-
return platforms
44-
45-
4631
docker_image_names = {
4732
"debian": "debian",
4833
"el": "centos",

‎packaging_automation/common_tool_methods.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,21 @@
3636
# all resources after the code execution.
3737
referenced_repos: List[Repo] = []
3838

39+
supported_platforms = {
40+
"debian": ["bullseye", "buster", "stretch", "jessie", "wheezy"],
41+
"el": ["8", "7", "6"],
42+
"ol": ["7", "8"],
43+
"ubuntu": ["focal", "bionic", "xenial", "trusty"]
44+
}
45+
46+
def platform_names() -> List[str]:
47+
platforms = []
48+
for platform_os, platform_releases in supported_platforms.items():
49+
for platform_release in platform_releases:
50+
platforms.append(f"{platform_os}/{platform_release}")
51+
platforms.append("pgxn")
52+
return platforms
53+
3954

4055
def get_new_repo(working_dir: str) -> Repo:
4156
repo = Repo(working_dir)

‎packaging_automation/get_postgres_versions.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import argparse
2+
import json
3+
4+
from .test_citus_package import (get_postgres_versions_from_matrix_file)
5+
6+
if __name__ == "__main__":
7+
parser = argparse.ArgumentParser()
8+
parser.add_argument('--prj_ver', required=True)
9+
10+
args = parser.parse_args()
11+
postgres_versions = get_postgres_versions_from_matrix_file(args.prj_ver)
12+
print(json.dumps(postgres_versions))

‎packaging_automation/test_citus_package.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import argparse
2+
import os
3+
import subprocess
4+
import shlex
5+
import requests
6+
from enum import Enum
7+
import sys
8+
from typing import List
9+
10+
from .common_tool_methods import (get_supported_postgres_release_versions, get_minor_version)
11+
12+
POSTGRES_MATRIX_FILE = "postgres-matrix.yml"
13+
POSTGRES_MATRIX_WEB_ADDRESS = "https://raw.githubusercontent.com/citusdata/packaging/all-citus-unit-tests/postgres-matrix.yml"
14+
15+
16+
def run_command(command: str) -> int:
17+
with subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process:
18+
for line in iter(process.stdout.readline, b''): # b'\n'-separated lines
19+
print(line.decode("utf-8"), end=" ")
20+
exitcode = process.wait()
21+
return exitcode
22+
23+
24+
class TestPlatform(Enum):
25+
el_7 = {"name": "el/7", "docker_image_name": "el-7"}
26+
el_8 = {"name": "el/8", "docker_image_name": "el-8"}
27+
centos_8 = {"name": "centos/8", "docker_image_name": "centos-8"}
28+
centos_7 = {"name": "centos/7", "docker_image_name": "centos-7"}
29+
ol_7 = {"name": "ol/7", "docker_image_name": "ol-7"}
30+
ol_8 = {"name": "ol/8", "docker_image_name": "ol-8"}
31+
debian_buster = {"name": "debian/buster", "docker_image_name": "debian-buster", }
32+
debian_bullseye = {"name": "debian/bullseye", "docker_image_name": "debian-bullseye"}
33+
debian_stretch = {"name": "debian/stretch", "docker_image_name": "debian-stretch"}
34+
ubuntu_bionic = {"name": "ubuntu/bionic", "docker_image_name": "ubuntu-bionic"}
35+
ubuntu_focal = {"name": "ubuntu/focal", "docker_image_name": "ubuntu-focal"}
36+
undefined = {"name": "undefined", "docker_image_name": "undefined"}
37+
38+
39+
def get_test_platform_for_os_release(os_release: str) -> TestPlatform:
40+
result = TestPlatform.undefined
41+
for tp in TestPlatform:
42+
if tp.value["name"] == os_release:
43+
result = tp
44+
return result
45+
46+
47+
def get_postgres_versions_from_matrix_file(project_version: str) -> List[str]:
48+
r = requests.get(POSTGRES_MATRIX_WEB_ADDRESS, allow_redirects=True)
49+
50+
with open(POSTGRES_MATRIX_FILE, 'wb') as writer:
51+
writer.write(r.content)
52+
pg_versions = get_supported_postgres_release_versions(POSTGRES_MATRIX_FILE, project_version)
53+
54+
return pg_versions
55+
56+
57+
if __name__ == "__main__":
58+
parser = argparse.ArgumentParser()
59+
parser.add_argument('--prj_ver', required=True)
60+
parser.add_argument('--pg_major_version')
61+
parser.add_argument("--os_release", choices=[t.value["name"] for t in TestPlatform])
62+
63+
args = parser.parse_args()
64+
test_platform = get_test_platform_for_os_release(args.os_release)
65+
minor_prj_ver = get_minor_version(args.prj_ver)
66+
67+
platform = args.os_release
68+
69+
postgres_versions = get_postgres_versions_from_matrix_file(args.prj_ver)
70+
71+
print(f'This version of Citus supports following pg versions: {postgres_versions}')
72+
73+
os.chdir("test-images")
74+
return_codes = {}
75+
76+
if args.pg_major_version:
77+
postgres_versions = [p for p in postgres_versions if p == args.pg_major_version]
78+
79+
80+
if len(postgres_versions) == 0:
81+
raise ValueError("At least one supported postgres version is required")
82+
83+
for postgres_version in postgres_versions:
84+
print(f'Testing package for following pg version: {postgres_version}')
85+
docker_image_name = f"test:{test_platform.value['docker_image_name']}-{postgres_version}"
86+
build_command = (f"docker build -t {docker_image_name} "
87+
f"-f {test_platform.value['docker_image_name']}/Dockerfile "
88+
f"--build-arg CITUS_VERSION={args.prj_ver} --build-arg PG_MAJOR={postgres_version} "
89+
f"--build-arg CITUS_MAJOR_VERSION={minor_prj_ver} .")
90+
print(build_command)
91+
return_build = run_command(build_command)
92+
return_run = run_command(
93+
f"docker run -e POSTGRES_VERSION={postgres_version} {docker_image_name} ")
94+
return_codes[f"{docker_image_name}-build"] = return_build
95+
return_codes[f"{docker_image_name}-run"] = return_run
96+
97+
error_exists = False
98+
print("-----------------Summary Report------------------")
99+
for key, value in return_codes.items():
100+
if value > 0:
101+
error_exists = True
102+
print(f"{key}: {'Success' if value == 0 else f'Fail. ErrorCode: {value}'}")
103+
summary_error = 'FAILED :(' if error_exists else 'SUCCESS :)'
104+
print(f'------------------------{summary_error}------------------------')
105+
106+
if error_exists:
107+
sys.exit("Failed")

‎test-images/centos-7/Dockerfile‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
FROM centos:7
2+
3+
ARG CITUS_VERSION
4+
# Format should be XY and should not include dots e.g for 10.2.1=>102
5+
ARG CITUS_MAJOR_VERSION
6+
ARG PG_MAJOR
7+
ARG FANCY=1
8+
ARG HLL_VERSION=2.16.citus-1
9+
ARG TOPN_VERSION=2.4.0.citus-1
10+
ARG PACKAGE_RELEASE_SUFFIX=el7
11+
12+
ENV CITUS_VERSION ${CITUS_VERSION}
13+
14+
ENV PG_MAJOR ${PG_MAJOR}
15+
16+
17+
RUN yum update -y && \
18+
yum install -y curl
19+
20+
21+
# TODO Parameterize Citus and postgres version
22+
RUN export CITUS_MAJOR_VER=${CITUS_MAJOR_VERSION//./} && \
23+
curl https://install.citusdata.com/community/rpm.sh | bash && \
24+
yum install -y citus${CITUS_MAJOR_VER}_${PG_MAJOR}-${CITUS_VERSION}.citus-${FANCY}.${PACKAGE_RELEASE_SUFFIX} \
25+
hll_${PG_MAJOR}-${HLL_VERSION}.${PACKAGE_RELEASE_SUFFIX} \
26+
topn_${PG_MAJOR}-${TOPN_VERSION}.${PACKAGE_RELEASE_SUFFIX}
27+
28+
29+
30+
ARG POSTGRES_HOME=/var/lib/pgsql
31+
ENV PATH=/usr/pgsql-${PG_MAJOR}/bin:${PATH}:${POSTGRES_HOME}
32+
33+
WORKDIR ${POSTGRES_HOME}
34+
35+
RUN mkdir citus && chown postgres citus
36+
37+
38+
39+
40+
41+
USER postgres
42+
RUN cd ~ && initdb -D citus && echo "shared_preload_libraries = 'citus'" >> citus/postgresql.conf
43+
44+
USER root
45+
# Install python 3.8 and its dependencies
46+
RUN yum install -y gcc make && \
47+
yum -y install libcurl-devel \
48+
openssl-devel \
49+
bzip2-devel \
50+
libffi-devel \
51+
xz-devel \
52+
python38-devel \
53+
openssl-devel &&\
54+
curl https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz --output Python-3.8.12.tgz &&\
55+
tar xvf Python-3.8.12.tgz &&\
56+
cd Python-3.8.*/ && \
57+
./configure --enable-optimizations && \
58+
make altinstall && \
59+
python3.8 -m pip install pip-tools
60+
61+
COPY scripts/* ./
62+
63+
RUN pip-compile && python3.8 -m pip install -r requirements.txt
64+
65+
66+
USER postgres
67+
68+
WORKDIR ${POSTGRES_HOME}
69+
70+
CMD ["test_internal.sh"]
71+

‎test-images/centos-8/Dockerfile‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
FROM centos:8
2+
3+
ARG CITUS_VERSION
4+
# Format should be XY and should not include dots e.g for 10.2.1=>102
5+
ARG CITUS_MAJOR_VERSION
6+
ARG PG_MAJOR
7+
ARG FANCY=1
8+
ARG HLL_VERSION=2.16.citus-1
9+
ARG TOPN_VERSION=2.4.0.citus-1
10+
ARG PACKAGE_RELEASE_SUFFIX=el8
11+
12+
ENV CITUS_VERSION ${CITUS_VERSION}
13+
14+
ENV PG_MAJOR ${PG_MAJOR}
15+
16+
17+
RUN yum update -y && \
18+
yum install -y curl
19+
20+
21+
# TODO Parameterize Citus and postgres version
22+
RUN export CITUS_MAJOR_VER=${CITUS_MAJOR_VERSION//./} && \
23+
curl https://install.citusdata.com/community/rpm.sh | bash && \
24+
yum install -y citus${CITUS_MAJOR_VER}_${PG_MAJOR}-${CITUS_VERSION}.citus-${FANCY}.${PACKAGE_RELEASE_SUFFIX} \
25+
hll_${PG_MAJOR}-${HLL_VERSION}.${PACKAGE_RELEASE_SUFFIX} \
26+
topn_${PG_MAJOR}-${TOPN_VERSION}.${PACKAGE_RELEASE_SUFFIX}
27+
28+
29+
30+
ARG POSTGRES_HOME=/var/lib/pgsql
31+
ENV PATH=/usr/pgsql-${PG_MAJOR}/bin:${PATH}:${POSTGRES_HOME}
32+
33+
WORKDIR ${POSTGRES_HOME}
34+
35+
RUN mkdir citus && chown postgres citus
36+
37+
38+
39+
40+
41+
USER postgres
42+
RUN cd ~ && initdb -D citus && echo "shared_preload_libraries = 'citus'" >> citus/postgresql.conf
43+
44+
USER root
45+
# Install python 3.8 and its dependencies
46+
RUN yum install -y gcc make && \
47+
yum -y install libcurl-devel \
48+
openssl-devel \
49+
bzip2-devel \
50+
libffi-devel \
51+
xz-devel \
52+
python38-devel \
53+
openssl-devel &&\
54+
curl https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz --output Python-3.8.12.tgz &&\
55+
tar xvf Python-3.8.12.tgz &&\
56+
cd Python-3.8.*/ && \
57+
./configure --enable-optimizations && \
58+
make altinstall && \
59+
python3.8 -m pip install pip-tools
60+
61+
COPY scripts/* ./
62+
63+
RUN pip-compile && python3.8 -m pip install -r requirements.txt
64+
65+
66+
USER postgres
67+
68+
WORKDIR ${POSTGRES_HOME}
69+
70+
CMD ["test_internal.sh"]
71+

0 commit comments

Comments
 (0)