Create a deb package (#25)

This adds the necessary configuration and CI to build the CLI as a deb package. This is accomplished using dh-virtualenv, which allows us to package Python dependencies in a virtual environment. Using dh-virtualenv not only allows us to use our own versions of Python dependencies, it also isolates our dependencies from the rest of the system. This allows us to have our own isolated installation of Docker Compose as a dependency, whose version we can update at our own pace.

The deb also obsoletes the need to install curl and rsync for the user, since they can now be expressed as package dependencies. However, Docker is currently still being installed for the user. We could consider using Ubuntu's version of Docker if we're comfortable with supporting an older version.

This includes an overhaul of our configuration system, so that different distributions of the CLI can be distributed with different defaults. Defaults are now provided by a defaults.yaml file. The deb package is given its own custom defaults.yaml that is configured to use system package locations like /var/lib/brainframe instead of user-installed locations like /var/local/brainframe.

Unfortunately, dh-virtualenv requires that we have a setup.py because it runs python3 setup.py install to add the application code to the virtual environment. I've tried to keep the setup.py as minimal as possible, shelling out to Poetry when I can.

This change is, admittedly, not immediately useful on its own. We would need a PPA or our own repository to distribute this deb to users, and updated instructions to go along with it. We will probably want to provide a script similar to get-docker.sh that handles all the installation steps for the user so that we can preserve our single-line installation experience.

See #13, which this PR does not resolve on its own.

Author: velovix

.circleci/config.yml

@@ -1,17 +1,36 @@
 version: 2.1
 
 parameters:
-  poetry_version:
+  poetry-version:
     type: string
     default: "1.1.4"
 
+
+executors:
+  focal:
+    docker:
+      - image: cimg/base:stable-20.04
+  bionic:
+    docker:
+      - image: cimg/base:stable-18.04
+
+
 workflows:
   lint:
     jobs:
       - lint
+  deb:
+    jobs:
+      - build-deb:
+          matrix:
+            parameters:
+              os: [bionic, focal]
+      - test-installation-deb:
+          requires:
+            - build-deb
   deploy:
     jobs:
-      - upload_to_pypi:
+      - upload-to-pypi:
           filters:
             # Ignore any commit on any branch by default
             branches:
@@ -21,60 +40,114 @@ workflows:
               only: /^v.+\..+\..+/
   tests:
     jobs:
-      - test_installation
+      - test-installation-source
 
 
 jobs:
   lint:
     docker:
-      - image: circleci/python:3.6
+      - image: cimg/python:3.6
     steps:
       - checkout
-      - run: "curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py
-          | python3 - --version << pipeline.parameters.poetry_version >>"
       - run: poetry install --no-root
       - run: poetry run isort --check .
       - run: poetry run black --check .
       - run: poetry run mypy -p "brainframe.cli"
-  upload_to_pypi:
+
+  build-deb:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y software-properties-common dpkg-dev devscripts equivs
+      - run: sudo add-apt-repository ppa:jyrki-pulliainen/dh-virtualenv
+      - run: sudo apt-get install -y dh-virtualenv
+      - install-poetry
+      - checkout
+      - run: cp -r package/debian .
+      - run: cp package/setup.py .
+      - run: cp package/system_package_defaults.yaml brainframe/cli/defaults.yaml
+      - run: sudo mk-build-deps --install debian/control
+      - run: poetry export --output requirements.txt
+      - run: dpkg-buildpackage --unsigned-source --unsigned-changes --build=binary
+      - run: mkdir dist
+      - run: |
+          export CODENAME=$(lsb_release --codename --short)
+          mv ../brainframe-cli*.deb dist/brainframe-cli-${CODENAME}.deb
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist/*
+
+  test-installation-deb:
+    machine:
+      image: ubuntu-2004:202010-01
+    steps:
+      - run: sudo apt-get update
+      - attach_workspace:
+          at: /tmp/workspace
+      - run: sudo apt-get install /tmp/workspace/dist/brainframe-cli-focal.deb
+      - run: sudo brainframe install --noninteractive
+      - run: sudo brainframe compose up -d
+      - assert-core-container-running
+      - assert-core-responds-to-http
+      - run: sudo brainframe compose down
+
+  upload-to-pypi:
     docker:
-      - image: circleci/python:3.6
+      - image: cimg/python:3.6
     environment:
       POETRY_HTTP_BASIC_PYPI_USERNAME: __token__
-
     steps:
       - checkout
-      - run: "curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py
-          | python3 - --version << pipeline.parameters.poetry_version >>"
       - run: poetry build
       - run: POETRY_HTTP_BASIC_PYPI_PASSWORD=${PYPI_PASSWORD} poetry publish
-  test_installation:
+
+  test-installation-source:
     machine:
       image: ubuntu-2004:202010-01
     steps:
       - run: sudo apt-get update
       - run: sudo apt-get install python3 python3-dev curl git -y
       - checkout
-      - run: "curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py
-          | python3 - --version << pipeline.parameters.poetry_version >>"
-      - run: echo "export PATH=$PATH:$HOME/.poetry/bin" >> $BASH_ENV
+      - install-poetry
       - run: sudo $(which poetry) install
       - run: sudo $(which poetry) run brainframe install --noninteractive
       - run: sudo $(which poetry) run brainframe compose up -d
+      - assert-core-container-running
+      - assert-core-responds-to-http
+      - run: sudo $(which poetry) run brainframe compose down
+
+commands:
+  install-poetry:
+    steps:
+      - run: sudo apt-get update && sudo apt-get install -y python3-pip
+      - run: >
+          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py
+          | python3 - --version << pipeline.parameters.poetry-version >>
+      # Add to the PATH
+      - run: echo 'export PATH=$HOME/.poetry/bin:$PATH' >> $BASH_ENV
+  assert-core-container-running:
+    steps:
       - run:
-          name: Check BrainFrame containers are running
+          name: Check BrainFrame core container is running
           command: |
             until docker container inspect -f {{.State.Running}}  $(docker ps -q -f name="brainframe_core") >/dev/null 2>/dev/null; do
                 sleep 0.1;
             done;
             echo "BrainFrame core container is running."
           no_output_timeout: 1m
+  assert-core-responds-to-http:
+    steps:
       - run:
-          name: Check BrainFrame core service is up
+          name: Check BrainFrame core responds to HTTP requests
           command: |
             until curl -f http://localhost/api/version >/dev/null 2>/dev/null; do
               sleep 0.1;
             done;
             echo "BrainFrame core service is up."
           no_output_timeout: 1m
-      - run: sudo $(which poetry) run brainframe compose down

brainframe/cli/commands/backup.py

@@ -5,9 +5,9 @@
 
 import i18n
 from brainframe.cli import (
+    config,
     dependencies,
     docker_compose,
-    env_vars,
     os_utils,
     print_utils,
 )
@@ -19,8 +19,8 @@
 
 @command("backup")
 def backup():
-    install_path = env_vars.install_path.get()
-    data_path = env_vars.data_path.get()
+    install_path = config.install_path.value
+    data_path = config.data_path.value
 
     args = _parse_args(data_path)
 

brainframe/cli/commands/compose.py

@@ -1,13 +1,12 @@
 import sys
 
-import i18n
-from brainframe.cli import docker_compose, env_vars, os_utils, print_utils
+from brainframe.cli import config, docker_compose
 
 from .utils import command
 
 
 @command("compose")
 def compose():
-    install_path = env_vars.install_path.get()
+    install_path = config.install_path.value
     docker_compose.assert_installed(install_path)
     docker_compose.run(install_path, sys.argv[2:])

brainframe/cli/commands/info.py

@@ -1,7 +1,7 @@
 from argparse import ArgumentParser
 
 import i18n
-from brainframe.cli import docker_compose, env_vars, print_utils
+from brainframe.cli import config, docker_compose, print_utils
 
 from .utils import command, subcommand_parse_args
 
@@ -10,15 +10,15 @@
 def info():
     args = _parse_args()
 
-    docker_compose.assert_installed(env_vars.install_path.get())
+    docker_compose.assert_installed(config.install_path.value)
 
     server_version = docker_compose.check_existing_version(
-        env_vars.install_path.get()
+        config.install_path.value
     )
 
     fields = {
-        "install_path": env_vars.install_path.get(),
-        "data_path": env_vars.data_path.get(),
+        "install_path": config.install_path.value,
+        "data_path": config.data_path.value,
         "server_version": server_version,
     }
 

brainframe/cli/commands/install.py

@@ -3,9 +3,9 @@
 
 import i18n
 from brainframe.cli import (
+    config,
     dependencies,
     docker_compose,
-    env_vars,
     os_utils,
     print_utils,
 )
@@ -32,9 +32,6 @@ def install():
     # Check all dependencies
     dependencies.curl.ensure(args.noninteractive, args.install_curl)
     dependencies.docker.ensure(args.noninteractive, args.install_docker)
-    dependencies.docker_compose.ensure(
-        args.noninteractive, args.install_docker_compose
-    )
 
     _, _, download_version = docker_compose.check_download_version()
     print_utils.translate("install.install-version", version=download_version)
@@ -55,8 +52,8 @@ def install():
         # Ask the user if they want to specify special paths for installation
         print_utils.translate(
             "install.default-paths",
-            default_install_path=env_vars.install_path.default,
-            default_data_path=env_vars.data_path.default,
+            default_install_path=config.install_path.default,
+            default_data_path=config.data_path.default,
         )
         use_default_paths = print_utils.ask_yes_no(
             "install.ask-use-default-paths",
@@ -66,22 +63,21 @@ def install():
     if args.noninteractive:
         install_path = args.install_path
     elif use_default_paths:
-        install_path = env_vars.install_path.default
+        install_path = config.install_path.default
     else:
         install_path = print_utils.ask_path(
-            "install.ask-brainframe-install-path",
-            env_vars.install_path.default,
+            "install.ask-brainframe-install-path", config.install_path.default,
         )
     install_path.mkdir(exist_ok=True, parents=True)
 
     # Set up the data path
     if args.noninteractive:
         data_path = args.data_path
     elif use_default_paths:
-        data_path = env_vars.data_path.default
+        data_path = config.data_path.default
     else:
         data_path = print_utils.ask_path(
-            "install.ask-data-path", env_vars.data_path.default
+            "install.ask-data-path", config.data_path.default
         )
     data_path.mkdir(exist_ok=True, parents=True)
 
@@ -120,15 +116,15 @@ def install():
     # Recommend to the user to add their custom paths to environment variables
     # so that future invocations of the program will know where to look.
     if (
-        install_path != env_vars.install_path.default
-        or data_path != env_vars.data_path.default
+        install_path != config.install_path.default
+        or data_path != config.data_path.default
     ):
         print()
         print_utils.translate("install.set-custom-directory-env-vars")
         print(
             f"\n"
-            f'export {env_vars.install_path.name}="{install_path}"\n'
-            f'export {env_vars.data_path.name}="{data_path}"\n'
+            f'export {config.install_path.name}="{install_path}"\n'
+            f'export {config.data_path.name}="{data_path}"\n'
         )
 
 
@@ -146,25 +142,20 @@ def _parse_args():
     parser.add_argument(
         "--install-path",
         type=Path,
-        default=env_vars.install_path.default,
+        default=config.install_path.default,
         help=i18n.t("install.install-path-help"),
     )
     parser.add_argument(
         "--data-path",
         type=Path,
-        default=env_vars.data_path.default,
+        default=config.data_path.default,
         help=i18n.t("install.data-path-help"),
     )
     parser.add_argument(
         "--install-docker",
         action="store_true",
         help=i18n.t("install.install-docker-help"),
     )
-    parser.add_argument(
-        "--install-docker-compose",
-        action="store_true",
-        help=i18n.t("install.install-docker-compose-help"),
-    )
     parser.add_argument(
         "--install-curl",
         action="store_true",

brainframe/cli/commands/update.py

@@ -1,7 +1,7 @@
 from argparse import ArgumentParser
 
 import i18n
-from brainframe.cli import docker_compose, env_vars, print_utils
+from brainframe.cli import config, docker_compose, print_utils
 from packaging import version
 
 from .utils import command, subcommand_parse_args
@@ -11,7 +11,7 @@
 def update():
     args = _parse_args()
 
-    install_path = env_vars.install_path.get()
+    install_path = config.install_path.value
 
     docker_compose.assert_installed(install_path)