Allow deployment of multiple sets of services (#289)

Update the compose file to allow easy deployment of multiple sets of
services:
 - ports are not bound by default, but through `compose.ports.yaml`
 - containers do not have a fixed name

Author: PGijsbers

.github/workflows/tests.yml

@@ -28,18 +28,18 @@ jobs:
       # https://github.com/docker/compose/issues/10596
       - name: Start services
         run: |
-          profiles="--profile python"
+          services="python-api"
           if [ "${{ matrix.php_api }}" = "true" ]; then
-            profiles="$profiles --profile php"
+            services="$services php-api"
           fi
-          docker compose $profiles up --detach --wait --remove-orphans || exit $(docker compose ps -q | xargs docker inspect -f '{{.State.ExitCode}}' | grep -v '^0' | wc -l)
+          docker compose up $services --detach --wait --remove-orphans || exit $(docker compose ps -q | xargs docker inspect -f '{{.State.ExitCode}}' | grep -v '^0' | wc -l)
 
       - name: Run tests
         run: |
           marker="${{ matrix.php_api == true && 'php_api' || 'not php_api' }} and ${{ matrix.mutations == true && 'mut' || 'not mut' }}"
-          docker exec openml-python-rest-api coverage run -m pytest -n auto -v -m "$marker"
+          docker compose exec python-api coverage run -m pytest -n auto -v -m "$marker"
       - name: Produce coverage report
-        run: docker exec openml-python-rest-api coverage xml
+        run: docker compose exec python-api coverage xml
       - name: Upload results to Codecov
         uses: codecov/codecov-action@v4
         with:

compose.ports.yaml

@@ -0,0 +1,21 @@
+services:
+  database:
+    ports:
+      - "3306:3306"
+
+  docs:
+    ports:
+      - "8000:8000"
+
+  elasticsearch:
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+
+  php-api:
+    ports:
+      - "8002:80"
+
+  python-api:
+    ports:
+      - "8001:8000"

docker-compose.yaml compose.yamldocker-compose.yaml renamed to compose.yaml

@@ -1,12 +1,8 @@
 services:
   database:
-    profiles: ["python", "php", "all"]
     image: "openml/test-database:v0.1.20260204"
-    container_name: "openml-test-database"
     environment:
       MYSQL_ROOT_PASSWORD: ok
-    ports:
-      - "3306:3306"
     healthcheck:
       test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
       start_period: 30s
@@ -20,19 +16,12 @@ services:
     build:
       context: .
       dockerfile: docker/docs/Dockerfile
-    ports:
-      - "8000:8000"
     volumes:
       - .:/docs
 
   elasticsearch:
-    profiles: ["php", "all"]
     image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23
-    container_name: "openml-elasticsearch"
     platform: "linux/amd64"
-    ports:
-      - "9200:9200"  # also known as /es (nginx)
-      - "9300:9300"
     env_file: docker/elasticsearch/.env
     healthcheck:
       test: curl 127.0.0.1:9200/_cluster/health | grep -e "green"
@@ -50,12 +39,9 @@ services:
           memory: 250M
 
   php-api:
-    profiles: ["php", "all"]
+    profiles: ["apis", "all"]
     image: "openml/php-rest-api:v1.2.4"
-    container_name: "openml-php-rest-api"
     env_file: docker/php/.env
-    ports:
-      - "8002:80"
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -69,13 +55,10 @@ services:
       interval: 1m
 
   python-api:
-    profiles: ["python", "all"]
-    container_name: "openml-python-rest-api"
+    profiles: ["apis", "all"]
     build:
       context: .
       dockerfile: docker/python/Dockerfile
-    ports:
-      - "8001:8000"
     volumes:
       - .:/app
       - ./src/config.toml:/config/config.toml

docs/contributing/contributing.md

@@ -33,31 +33,36 @@ required services for development is through [`docker compose`](https://docs.doc
 ```bash
 docker compose --profile all up -d
 ```
+or, to expose the services to the host network at the same time:
+```bash
+docker compose -f compose.yaml -f compose.ports.yaml --profile all up -d
+```
 
-This will spin up 5 containers, as defined in the `docker-compose.yaml` file:
+This will spin up 5 services, as defined in the `compose.yaml` file:
 
- - `openml-test-database`: this is a mysql database prepopulated with test data.
-    It is reachable from the host machine with port `3306`, by default it is configured
-    to have a root user with password `"ok"`.
- - `server-api-docs-1`: this container serves project documentation at `localhost:8000`.
+ - `database`: this is a mysql database prepopulated with test data.
+    By default it is configured to have a root user with password `"ok"`.
+ - `docs`: this container serves project documentation at `localhost:8000`.
     These pages are built from the documents in the `docs/` directory of this repository,
     whenever you edit and save a file there, the page will immediately be updated.
- - `openml-php-rest-api`: this container serves the old PHP REST API at `localhost:8002`.
+ - `elasticsearch`: Elasticsearch, required for the PHP REST API to function.
+ - `php-api`: this container serves the old PHP REST API at `localhost:8002`.
     For example, visit [http://localhost:8002/api/v1/json/data/1](http://localhost:8002/api/v1/json/data/1)
     to fetch a JSON description of dataset 1.
- - `openml-elasticsearch`: Elasticsearch, required for the PHP REST API to function.
- - `openml-python-rest-api`: this container serves the new Python-based REST API at `localhost:8001`.
+ - `python-api`: this container serves the new Python-based REST API at `localhost:8001`.
     For example, visit [http://localhost:8001/docs](http://localhost:8001/docs) to see
     the REST API documentation. Changes to the code in `src/` will be reflected in this
     container.
 
+Exposing ports to the host network isn't needed for development, but may be useful to inspect responses directly from the host machine.
+
 !!! note
     On arm-based Macs, you need to enable Rosetta emulation for Docker for the Elasticsearch container to work.
 
 We can now run the full test suite, which takes about 4 minutes:
 
 ```bash
-docker exec openml-python-rest-api python -m pytest tests
+docker compose exec python-api python -m pytest tests
 ```
 There are three important [test markers](https://docs.pytest.org/en/7.1.x/example/markers.html) to be aware of:
 
@@ -69,15 +74,15 @@ In many cases during development it's sufficient to either run with `not php_api
 The `not slow` is only needed if a slow test would be included in your test selection. In many cases, you might prefer to only run the specific tests (or test modules) that you are working on and excluding it through markers may be unnecessary.
 Examples:
 
- - `docker exec openml-python-rest-api python -m pytest tests -m "not php_api and not slow"`, here the test selection is made primarily through markers. This command takes a few seconds.
- - `docker exec openml-python-rest-api python -m pytest tests/routers/openml/dataset_tag_test.py`, here the test selection is made through specifying the file with tests. Since this test file naturally includes neither migration tests (in `tests/routers/openml/migration`) nor the slow test (at `tests/routers/openml/datasets_list_datasets_test.py`), excluding tests through markers is unnecessary. This command takes a few seconds.
+ - `docker compose exec python-api python -m pytest tests -m "not php_api and not slow"`, here the test selection is made primarily through markers. This command takes a few seconds.
+ - `docker compose exec python-api python -m pytest tests/routers/openml/dataset_tag_test.py`, here the test selection is made through specifying the file with tests. Since this test file naturally includes neither migration tests (in `tests/routers/openml/migration`) nor the slow test (at `tests/routers/openml/datasets_list_datasets_test.py`), excluding tests through markers is unnecessary. This command takes a few seconds.
 
 
 You don't always need every container, often just having a database and the Python-based
 REST API may be enough. In that case, only specify those services:
 
 ```bash
-docker compose --profile python up -d
+docker compose up python-api -d
 ```
 
 Refer to the `docker compose` documentation for more uses.
@@ -87,18 +92,18 @@ Refer to the `docker compose` documentation for more uses.
 
 ### Connecting to containers
 
-To connect to a container, run:
+To connect to a container of a service, run:
 
 ```bash
-docker exec -it CONTAINER_NAME /bin/bash
+docker compose exec SERVICE_NAME /bin/bash
 ```
 
-where `CONTAINER_NAME` is the name of the container. If you are unsure of your container
-name, then `docker container ls` may help you find it. Assuming the default container
+where `SERVICE_NAME` is the name of the service. If you are unsure of the service
+name, then `docker compose ps` may help you find it. Assuming the default service
 names are used, you may connect to the Python-based REST API container using:
 
 ```bash
-docker exec -it openml-python-rest-api /bin/bash
+docker compose exec python-api /bin/bash
 ```
 
 This is useful, for example, to run unit tests in the container:

docs/installation.md

@@ -4,15 +4,16 @@ See also ["Contributing"](contributing/contributing.md).
 
 The primary way to run this service is through a Docker container.
 The REST API needs to be able to connect to a MySQL database with the OpenML "openml" and "openml_expdb" databases.
-The `docker-compose.yaml` file of this project defines these together out of the box.
+The `compose.yaml` file of this project defines these together out of the box.
 This is useful for development purposes, but the database does not persist between restarts in the current configuration.
 By default, the current code is also mounted into the Python REST API container (again, for development purposes).
 
-For development, it should suffice to run the services from a fresh clone by running `docker compose --profile "python" up -d`.
-The REST API will be exposed on port 8001 on the host machine. To visit the Swagger Docs, visit http://localhost:8001/docs.
+For development, it should suffice to run the services from a fresh clone by running `docker compose up python-api -d`.
+If you want to make sure to bind the exposed container ports to the host machine then you will need to use the `compose.ports.yaml` file too (`docker compose -f compose.yaml -f compose.ports.yaml up python-api -d`).
+The REST API will then be exposed on port 8001 on the host machine. To visit the Swagger Docs, visit http://localhost:8001/docs.
 
-Once the containers are started, you can run tests with `docker exec -it openml-python-rest-api python -m pytest -m "not php_api" tests`.
-For migration testing, which compares output of the Python-based REST API with the old PHP-based one, also start the PHP server (`docker compose --profile "php" --profile "python" up -d`) and include tests with the `php_api` marker/fixture: `docker exec -it openml-python-rest-api python -m pytest tests`.
+Once the containers are started, you can run tests with `docker compose exec python-api python -m pytest -m "not php_api" tests`.
+For migration testing, which compares output of the Python-based REST API with the old PHP-based one, also start the PHP server (`docker compose --profile "apis" up -d`) and include tests with the `php_api` marker/fixture: `docker compose exec python-api python -m pytest tests`.
 
 !!! note
 
@@ -21,4 +22,4 @@ For migration testing, which compares output of the Python-based REST API with t
     When we start testing more upload functionality, for which the PHP API needs built indices, we'll work on an ES image with prebuilt indices.
 
 Information for a production deployment will follow, in a nutshell you need to configure the REST API to connect to a persistent database,
-which can be the one defined in `docker-compose.yaml` if has an appropriately mounted volume.
+which can be the one defined in `compose.yaml` if it has an appropriately mounted volume.

src/config.toml

@@ -8,7 +8,7 @@ allow_test_api_keys=true
 root_path=""
 
 [databases.defaults]
-host="openml-test-database"
+host="database"
 port="3306"
 # SQLAlchemy `dialect` and `driver`: https://docs.sqlalchemy.org/en/20/dialects/index.html
 drivername="mysql+aiomysql"

tests/conftest.py

@@ -16,7 +16,7 @@
 from main import create_api
 from routers.dependencies import expdb_connection, userdb_connection
 
-PHP_API_URL = "http://openml-php-rest-api:80/api/v1/json"
+PHP_API_URL = "http://php-api:80/api/v1/json"
 
 
 @contextlib.asynccontextmanager