feat(services): move services setup to task model + remove Python (#34)

README.md

@@ -7,9 +7,9 @@ anyone else.
 
 ## Getting started
 
-Use `. script/bootstrap` to set up the Python environment needed for the invoke and pyinfra tooling to work.
+Use `. bootstrap.sh` to set up Task.
 
-This expects `pyenv` to be set up on your system.
+Once this is done, `task -l` will outline all available commands.
 
 ## Configuration
 
@@ -30,6 +30,8 @@ env:
     ENV_FILE_DIR: ...
     # Name of the network associated with the tunnel exposing services.
     SHARED_NETWORK_NAME: ...
+    # Path to the storage root.
+    STORAGE_DIR: ...
 ```
 
 The `env.yml` file is ignored by version-control.

pyinfra/inventory.py

@@ -1 +0,0 @@
-hosts = [("spadinaistan.karnov.club", {"ssh_port": 6969})]

pyinfra/reboot.py

@@ -1,14 +0,0 @@
-"""
-Reboot machines.
-"""
-
-import pyinfra
-
-
-def reboot_machines():
-    pyinfra.operations.server.reboot(
-        name="Reboot server", _sudo=True, _use_sudo_password=True
-    )
-
-
-reboot_machines()

pyinfra/system_updates.py

@@ -1,20 +0,0 @@
-"""
-Runs Ubuntu system updates.
-"""
-
-import pyinfra
-
-
-def update_ubuntu_packages():
-    common = {"_sudo": True, "_use_sudo_password": True}
-    pyinfra.operations.server.shell(
-        name="Search for system updates", commands=["apt update"], **common
-    )
-    pyinfra.operations.server.shell(
-        name="Apply updates and remove orphaned packages",
-        commands=["apt upgrade --autoremove -y"],
-        **common
-    )
-
-
-update_ubuntu_packages()

requirements.in

@@ -1,5 +0,0 @@
-black
-invoke
-pyinfra
-pyyaml ~= 6.0.0
-jinja2 ~= 3.1.0

requirements.txt

@@ -1,93 +0,0 @@
-#
-# This file is autogenerated by pip-compile with Python 3.11
-# by the following command:
-#
-#    pip-compile ./requirements.in
-#
-bcrypt==4.0.1
-    # via paramiko
-black==23.3.0
-    # via -r ./requirements.in
-certifi==2022.12.7
-    # via requests
-cffi==1.15.1
-    # via
-    #   cryptography
-    #   pynacl
-charset-normalizer==3.1.0
-    # via requests
-click==8.1.3
-    # via
-    #   black
-    #   pyinfra
-colorama==0.4.6
-    # via pyinfra
-configparser==5.3.0
-    # via pyinfra
-cryptography==40.0.2
-    # via
-    #   paramiko
-    #   pyspnego
-    #   requests-ntlm
-distro==1.8.0
-    # via pyinfra
-gevent==22.10.2
-    # via pyinfra
-greenlet==2.0.2
-    # via gevent
-idna==3.4
-    # via requests
-invoke==2.1.0
-    # via -r ./requirements.in
-jinja2==3.1.2
-    # via
-    #   -r ./requirements.in
-    #   pyinfra
-markupsafe==2.1.2
-    # via jinja2
-mypy-extensions==1.0.0
-    # via black
-packaging==23.1
-    # via black
-paramiko==2.12.0
-    # via pyinfra
-pathspec==0.11.1
-    # via black
-platformdirs==3.5.0
-    # via black
-pycparser==2.21
-    # via cffi
-pyinfra==2.6.2
-    # via -r ./requirements.in
-pynacl==1.5.0
-    # via paramiko
-pyspnego==0.9.0
-    # via requests-ntlm
-python-dateutil==2.8.2
-    # via pyinfra
-pywinrm==0.4.3
-    # via pyinfra
-pyyaml==6.0.1
-    # via -r ./requirements.in
-requests==2.29.0
-    # via
-    #   pywinrm
-    #   requests-ntlm
-requests-ntlm==1.2.0
-    # via pywinrm
-six==1.16.0
-    # via
-    #   paramiko
-    #   python-dateutil
-    #   pywinrm
-urllib3==1.26.15
-    # via requests
-xmltodict==0.13.0
-    # via pywinrm
-zope-event==4.6
-    # via gevent
-zope-interface==6.0
-    # via gevent
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

script/bootstrap

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-VENV="spadinaistan.venv"
-
-python -m pip install pip~=23.1 pip-tools==6.13.0 --no-cache
-
-if [ ! -d "./$VENV" ]; then
-    python -m venv ./$VENV
-fi
-
-source ./$VENV/bin/activate
-
-
-pip-sync ./requirements.txt

script/lock

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-pip-compile ./requirements.in
-

services.yml.j2

@@ -1,47 +0,0 @@
-services:
-  deluge:
-    ports:
-      - 8112:8112
-      - 6881:6881/udp
-      - 6881:6881/tcp
-    volumes:
-      - {{ DELUGE_CONFIG_ROOT }}:/config
-      - {{ DELUGE_DOWNLOADS_ROOT }}:/downloads
-      - {{ DELUGE_COMPLETE_DOWNLOADS_ROOT }}:/complete
-    env_files:
-      - {{ ENVS_ROOT }}/deluge.env
-  bastion:
-    env_files:
-      - {{ ENVS_ROOT }}/bastion.env
-  bitwarden:
-    env_files:
-      - {{ ENVS_ROOT }}/bitwarden-web.env
-    ports:
-      - 8080:8080
-    volumes:
-      - {{ BITWARDEN_CONFIG_ROOT }}:/etc/bitwarden
-      - {{ BITWARDEN_LOGS_ROOT }}:/var/log/bitwarden
-  bitwarden-db:
-    env_files:
-      - {{ ENVS_ROOT }}/bitwarden-db.env
-    volumes:
-      - {{ BITWARDEN_DB_ROOT }}:/var/lib/mysql
-    ports:
-      - 3306:3306
-  plex:
-    ports:
-      - published=32400,target=32400,mode=host,protocol=tcp
-      - 32469:32469/tcp
-      - 3005:3005/tcp
-      - 8324:8324/tcp
-      - 1900:1900/udp
-      - 32410:32410/udp
-      - 32412:32412/udp
-      - 32413:32413/udp
-      - 32414:32414/udp
-    env_files:
-      - {{ ENVS_ROOT }}/plex.env
-    volumes:
-      - {{ PLEX_DB_ROOT }}:/config
-      - {{ PLEX_TRANSCODE_ROOT }}:/transcode
-      - {{ PLEX_DATA_ROOT }}:/data

services/bastion/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source constants.sh
+
+docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-bastion-app .

services/bastion/constants.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export APP_NAME="bastion"
+export APP_CONTAINER_NAME=$APP_NAME-app
+export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME
+export NETWORK_NAME=$APP_NAME-local

services/bastion/start.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+source constants.sh
+
+docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists"
+
+docker run \
+    --detach \
+    --network $NETWORK_NAME \
+    --name $APP_CONTAINER_NAME \
+    --env-file $ENV_FILE_DIR/bastion.env \
+    $APP_IMAGE_NAME:dev

services/bastion/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source constants.sh
+
+docker rm -f $APP_CONTAINER_NAME

services/bitwarden/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source constants.sh
+
+docker build -t $DB_IMAGE_NAME:dev -f Dockerfile-bitwarden-db .
+docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-bitwarden-app .

services/bitwarden/constants.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export APP_NAME="bitwarden"
+export APP_CONTAINER_NAME=$APP_NAME-app
+export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME
+export DB_CONTAINER_NAME=$APP_NAME-db
+export DB_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$DB_CONTAINER_NAME
+export NETWORK_NAME=$APP_NAME-local

services/bitwarden/start.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+source constants.sh
+
+docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists"
+
+docker run \
+    --detach \
+    --network $NETWORK_NAME \
+    --name $DB_CONTAINER_NAME \
+    --env-file $ENV_FILE_DIR/bitwarden-db.env \
+    --mount type=bind,source=$APP_DATA_DIR/bitwarden/db,target=/var/lib/mysql \
+    $DB_IMAGE_NAME:dev
+
+docker run \
+    --detach \
+    --network $NETWORK_NAME \
+    --name $APP_CONTAINER_NAME \
+    --env-file $ENV_FILE_DIR/bitwarden-web.env \
+    --publish 8080:8080 \
+    --mount type=bind,source=$APP_DATA_DIR/bitwarden/config,target=/etc/bitwarden \
+    --mount type=bind,source=$APP_DATA_DIR/bitwarden/logs,target=/var/log/bitwarden \
+    $APP_IMAGE_NAME:dev

services/bitwarden/stop.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source constants.sh
+
+docker rm -f $APP_CONTAINER_NAME
+docker rm -f $DB_CONTAINER_NAME

services/deluge/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source constants.sh
+
+docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-deluge-app .

services/deluge/constants.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export APP_NAME="deluge"
+export APP_CONTAINER_NAME=$APP_NAME-app
+export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME
+export NETWORK_NAME=$APP_NAME-local

services/deluge/start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source constants.sh
+
+docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists"
+
+docker run \
+    --detach \
+    --network $NETWORK_NAME \
+    --name $APP_CONTAINER_NAME \
+    --env-file $ENV_FILE_DIR/deluge.env \
+    --publish 8112:8112 \
+    --publish 6881:6881/udp \
+    --publish 6881:6881/tcp \
+    --mount type=bind,source=$APP_DATA_DIR/deluge/config,target=/config \
+    --mount type=bind,source=$APP_DATA_DIR/deluge/downloads,target=/downloads \
+    --mount type=bind,source=$STORAGE_DIR,target=/complete \
+    $APP_IMAGE_NAME:dev

services/deluge/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source constants.sh
+
+docker rm -f $APP_CONTAINER_NAME

services/plex/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source constants.sh
+
+docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-plex-app .

services/plex/constants.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export APP_NAME="plex"
+export APP_CONTAINER_NAME=$APP_NAME-app
+export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME
+export DB_CONTAINER_NAME=$APP_NAME-db
+export DB_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$DB_CONTAINER_NAME
+export NETWORK_NAME=$APP_NAME-local

services/plex/start.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+source constants.sh
+
+docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists"
+
+docker run \
+    --detach \
+    --network $NETWORK_NAME \
+    --name $APP_CONTAINER_NAME \
+    --env-file $ENV_FILE_DIR/plex.env \
+    --publish 32401:32400 \
+    --publish 32469:32469 \
+    --publish 3005:3005 \
+    --publish 8324:8324 \
+    --publish 1900:1900/udp \
+    --publish 32410:32410/udp \
+    --publish 32412:32412/udp \
+    --publish 32413:32413/udp \
+    --publish 32414:32414/udp \
+    --mount type=bind,source=$APP_DATA_DIR/plex/database,target=/config \
+    --mount type=bind,source=$APP_DATA_DIR/plex/transcode,target=/transcode \
+    --mount type=bind,source=$STORAGE_DIR/media,target=/data \
+    $APP_IMAGE_NAME:dev

services/plex/stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source constants.sh
+
+docker rm -f $APP_CONTAINER_NAME

tasks.py

@@ -1,160 +0,0 @@
-import invoke
-import jinja2
-import yaml
-
-import os
-import pathlib
-import typing
-
-ns = invoke.Collection()
-
-PYINFRA_COMMON_PREFIX = "pyinfra -vvv pyinfra/inventory.py"
-
-
-def _get_tag(ctx: invoke.context.Context) -> str:
-    """
-    Gets the current tag, if it exists, for image versioning purposes.
-    """
-    try:
-        out = ctx.run("git describe --tags --exact-match", hide=True)
-        return out.stdout.strip()
-    except:
-        return "dev"
-
-
-def _ensure_networks(ctx: invoke.context.Context):
-    """
-    Ensures that the required networks exist.
-    """
-    print("Ensuring networks exist...")
-    output = ctx.run("docker network ls -f name=spad-internal -q", hide=True)
-    if not bool(output.stdout):
-        ctx.run("docker network create --scope=swarm spad-internal")
-        print('\t✅ Created network "internal".')
-    else:
-        print('\t✅ Network "internal" already exists.')
-
-
-def _ensure_swarm(ctx: invoke.context.Context):
-    """
-    Ensures that swarm mode is on.
-    """
-    print("Ensuring swarm mode is on...")
-    try:
-        ctx.run("docker swarm init --advertise-addr 192.168.1.17", hide=True)
-    except:
-        pass
-    print("\t✅ Swarm in on.")
-
-
-@invoke.task()
-def system_updates(ctx):
-    ctx.run(f"{PYINFRA_COMMON_PREFIX} pyinfra/system_updates.py")
-
-
-@invoke.task()
-def system_reboot(ctx):
-    ctx.run(f"{PYINFRA_COMMON_PREFIX} pyinfra/reboot.py")
-
-
-@invoke.task()
-def generate_configuration(ctx):
-    """
-    Generates the service configuration file `services.yml`
-    based on environment variables.
-
-    To avoid polluting the environment, this can be called as
-    `env $(cat .env | xargs) inv generate-configuration` where
-    .env contains the variables.
-    """
-    template_loader = jinja2.FileSystemLoader(searchpath="./")
-    template_env = jinja2.Environment(
-        loader=template_loader, undefined=jinja2.StrictUndefined
-    )
-    template = template_env.get_template("services.yml.j2")
-
-    with open("services.yml", "w") as outfile:
-        outfile.write(template.render(**os.environ))
-
-
-@invoke.task()
-def start(ctx, services: str):
-    """
-    Starts one of more services, defined by a comma-separated list of labels.
-
-    Services should correspond to an entry in `services.yml`.
-    """
-    _ensure_swarm(ctx)
-    _ensure_networks(ctx)
-
-    current_version = _get_tag(ctx)
-
-    print("Starting services...")
-
-    for service in services.split(","):
-        with open("services.yml", "r") as config:
-            service_config = yaml.load(config, Loader=yaml.Loader)["services"][service]
-
-        ctx.run(
-            f"docker build -t spadinaistan-{service}:{current_version} -f services/Dockerfile-{service} .",
-            hide=True,
-        )
-        ports_args = (
-            f"--publish {ports}\\\n" for ports in service_config.get("ports", [])
-        )
-
-        volume_args = []
-
-        for volume in service_config.get("volumes", []):
-            source, target = volume.split(":")
-            volume_args.append(f"--mount type=bind,source={source},target={target}\\\n")
-
-        env_files = (
-            f"--env-file {env_file}\\\n"
-            for env_file in service_config.get("env_files", [])
-        )
-
-        try:
-            out = ctx.run(
-                f"""docker service create \
-                            --name spad-{service}\
-                            --network spad-internal\
-                            {" ".join(env_files)}\
-                            --hostname=\"{service}\"\
-                            {" ".join(volume_args)}\
-                            {" ".join(ports_args)}\
-                            spadinaistan-{service}:{current_version}""",
-                hide=True,
-            )
-        except:
-            print(out.stdout)
-            print(out.stderr)
-            continue
-
-        print(f"\t✅ {service} is running.")
-
-
-@invoke.task()
-def stop(ctx: invoke.context.Context, services: str):
-    """
-    Stops the provided list of services, as a comma-separated list
-    of labels.
-    """
-    print("Stopping services...")
-    for service in services.split(","):
-        ctx.run(f"docker service rm spad-{service}", hide=True)
-        print(f"\t✅ {service} is stopped.")
-
-
-services = invoke.Collection("service")
-services.add_task(start)
-services.add_task(stop)
-
-server = invoke.Collection("server")
-
-server.add_task(system_updates, name="update")
-server.add_task(system_reboot, name="reboot")
-
-ns.add_collection(server)
-ns.add_collection(services)
-ns.add_task(generate_configuration)