infra: remove docker compose dependency (#28)

* infra(techdebt): migrate bastion, deluge to not use compose

* infra: migrate plex, move env files out of repository path

* infra: migrate bitwarden

* infra: add version tagging

* chore: linting + docs

.gitignore

@@ -1,5 +1,6 @@
 spadinaistan.venv
 
+services.yml
 **/*.env
 **/.env
 pyinfra-debug.log

requirements.in

@@ -1,3 +1,5 @@
 black
 invoke
 pyinfra
+pyyaml ~= 6.0.0
+jinja2 ~= 3.1.0

requirements.txt

@@ -40,7 +40,9 @@ idna==3.4
 invoke==2.1.0
     # via -r ./requirements.in
 jinja2==3.1.2
-    # via pyinfra
+    # via
+    #   -r ./requirements.in
+    #   pyinfra
 markupsafe==2.1.2
     # via jinja2
 mypy-extensions==1.0.0
@@ -65,6 +67,8 @@ python-dateutil==2.8.2
     # via pyinfra
 pywinrm==0.4.3
     # via pyinfra
+pyyaml==6.0.1
+    # via -r ./requirements.in
 requests==2.29.0
     # via
     #   pywinrm

services.yml.j2

@@ -0,0 +1,48 @@
+services:
+  deluge:
+    ports:
+      - 8112:8112
+      - 6881:6881/udp
+      - 6881:6881/tcp
+    volumes:
+      - {{ DELUGE_CONFIG_ROOT }}:/config
+      - {{ DELUGE_DOWNLOADS_ROOT }}:/downloads
+      - {{ DELUGE_COMPLETE_DOWNLOADS_ROOT }}:/complete
+    env_files:
+      - {{ ENVS_ROOT }}/deluge.env
+  bastion:
+    env_files:
+      - {{ ENVS_ROOT }}/bastion.env
+  bitwarden:
+    env_files:
+      - {{ ENVS_ROOT }}/bitwarden-web.env
+    port:
+      - 7000:8080
+      - 7001:8443
+    volumes:
+      - {{ BITWARDEN_CONFIG_ROOT }}:/etc/bitwarden
+      - {{ BITWARDEN_LOGS_ROOT }}:/var/log/bitwarden
+  bitwarden-db:
+    env_files:
+      - {{ ENVS_ROOT }}/bitwarden-db.env
+    volumes:
+      - {{ BITWARDEN_DB_ROOT }}:/var/lib/mysql
+    ports:
+      - 3306:3306
+  plex:
+    ports:
+      - 32400:32400/tcp
+      - 32469:32469/tcp
+      - 3005:3005/tcp
+      - 8324:8324/tcp
+      - 1900:1900/udp
+      - 32410:32410/udp
+      - 32412:32412/udp
+      - 32413:32413/udp
+      - 32414:32414/udp
+    env_files:
+      - {{ ENVS_ROOT }}/plex.env
+    volumes:
+      - {{ PLEX_DB_ROOT }}:/config
+      - {{ PLEX_TRANSCODE_ROOT }}:/transcode
+      - {{ PLEX_DATA_ROOT }}:/data

services/Dockerfile-bastion

@@ -0,0 +1,7 @@
+FROM cloudflare/cloudflared:1414-cb4bd8d06572
+
+ARG TUNNEL_SECRET
+
+ENV TUNNEL_TOKEN=$TUNNEL_SECRET
+
+CMD ["tunnel", "run"]

services/Dockerfile-bitwarden

@@ -0,0 +1 @@
+FROM bitwarden/self-host:beta

services/Dockerfile-bitwarden-db

@@ -0,0 +1 @@
+FROM mariadb:10

services/Dockerfile-deluge

@@ -0,0 +1,5 @@
+FROM lscr.io/linuxserver/deluge:latest
+
+ENV TZ=America/Toronto
+
+EXPOSE 8112 6881/tcp 6881/udp

services/Dockerfile-plex

@@ -0,0 +1,3 @@
+FROM plexinc/pms-docker:1.32.0.6973-a787c5a8e
+
+ENV TZ=America/Toronto

tasks.py

@@ -1,5 +1,8 @@
 import invoke
+import jinja2
+import yaml
 
+import os
 import pathlib
 import typing
 
@@ -8,6 +11,42 @@ ns = invoke.Collection()
 PYINFRA_COMMON_PREFIX = "pyinfra -vvv pyinfra/inventory.py"
 
 
+def _get_tag(ctx: invoke.context.Context) -> str:
+    """
+    Gets the current tag, if it exists, for image versioning purposes.
+    """
+    try:
+        out = ctx.run("git describe --tags --exact-match", hide=True)
+        return out.stdout.strip()
+    except:
+        return "dev"
+
+
+def _ensure_networks(ctx: invoke.context.Context):
+    """
+    Ensures that the required networks exist.
+    """
+    print("Ensuring networks exist...")
+    output = ctx.run("docker network ls -f name=spad-internal -q", hide=True)
+    if not bool(output.stdout):
+        ctx.run("docker network create --scope=swarm spad-internal")
+        print('\t✅ Created network "internal".')
+    else:
+        print('\t✅ Network "internal" already exists.')
+
+
+def _ensure_swarm(ctx: invoke.context.Context):
+    """
+    Ensures that swarm mode is on.
+    """
+    print("Ensuring swarm mode is on...")
+    try:
+        ctx.run("docker swarm init --advertise-addr 192.168.1.17", hide=True)
+    except:
+        pass
+    print("\t✅ Swarm in on.")
+
+
 @invoke.task()
 def system_updates(ctx):
     ctx.run(f"{PYINFRA_COMMON_PREFIX} pyinfra/system_updates.py")
@@ -19,79 +58,97 @@ def system_reboot(ctx):
 
 
 @invoke.task()
-def start(context: invoke.context.Context, service: typing.Optional[str]):
+def generate_configuration(ctx):
     """
-    Starts a service.
+    Generates the service configuration file `services.yml`
+    based on environment variables.
 
-    Services are assumed to be docker-compose-friendly and start as
+    To avoid polluting the environment, this can be called as
-    docker-compose up -d.
+    `env $(cat .env | xargs) inv generate-configuration` where
-
+    .env contains the variables.
-    The supplied service name is used to pathing, with the expected
-    file structure being
-    /
-      /services
-         /service1
-            docker-compose.yml
     """
+    template_loader = jinja2.FileSystemLoader(searchpath="./")
+    template_env = jinja2.Environment(
+        loader=template_loader, undefined=jinja2.StrictUndefined
+    )
+    template = template_env.get_template("services.yml.j2")
 
-    if service is None:
+    with open("services.yml", "w") as outfile:
-        raise ValueError("Service name must be provided.")
+        outfile.write(template.render(**os.environ))
-
-    service_path = pathlib.Path("services", service)
-
-    if not service_path.exists():
-        raise ValueError(f"Service path does not exist: {service_path}")
-
-    with context.cd(service_path):
-        context.run("docker-compose up --build --force-recreate -d")
 
 
 @invoke.task()
-def stop(context: invoke.context.Context, service: typing.Optional[str]):
+def start(ctx, services: str):
     """
-    Stops a service.
+    Starts one of more services, defined by a comma-separated list of labels.
 
-    The same assumptions about file and service structure as made as with
+    Services should correspond to an entry in `services.yml`.
-    `start <service>`.
     """
+    _ensure_swarm(ctx)
+    _ensure_networks(ctx)
 
-    if service is None:
+    current_version = _get_tag(ctx)
-        raise ValueError("Service name must be provided.")
 
-    service_path = pathlib.Path("services", service)
+    print("Starting services...")
 
-    if not service_path.exists():
+    for service in services.split(","):
-        raise ValueError(f"Service path does not exist: {service_path}")
+        with open("services.yml", "r") as config:
+            service_config = yaml.load(config, Loader=yaml.Loader)["services"][service]
 
-    with context.cd(service_path):
+        ctx.run(
-        context.run("docker-compose down")
+            f"docker build -t spadinaistan-{service}:{current_version} -f services/Dockerfile-{service} .",
+            hide=True,
+        )
+        ports_args = (
+            f"--publish {ports}\\\n" for ports in service_config.get("ports", [])
+        )
+
+        volume_args = []
+
+        for volume in service_config.get("volumes", []):
+            source, target = volume.split(":")
+            volume_args.append(f"--mount type=bind,source={source},target={target}\\\n")
+
+        env_files = (
+            f"--env-file {env_file}\\\n"
+            for env_file in service_config.get("env_files", [])
+        )
+
+        try:
+            out = ctx.run(
+                f"""docker service create \
+                            --name spad-{service}\
+                            --network spad-internal\
+                            {" ".join(env_files)}\
+                            --hostname=\"{service}\"\
+                            {" ".join(volume_args)}\
+                            {" ".join(ports_args)}\
+                            spadinaistan-{service}:{current_version}""",
+                hide=True,
+            )
+        except:
+            print(out.stdout)
+            print(out.stderr)
+            continue
+
+        print(f"\t✅ {service} is running.")
 
 
 @invoke.task()
-def restart(context: invoke.context.Context, service: typing.Optional[str]):
+def stop(ctx: invoke.context.Context, services: str):
     """
-    Restarts a service.
+    Stops the provided list of services, as a comma-separated list
-
+    of labels.
-    The same assumptions about file and service structure as made as with
-    `start <service>`.
     """
-
+    print("Stopping services...")
-    if service is None:
+    for service in services.split(","):
-        raise ValueError("Service name must be provided.")
+        ctx.run(f"docker service rm spad-{service}", hide=True)
-
+        print(f"\t✅ {service} is stopped.")
-    service_path = pathlib.Path("services", service)
-
-    if not service_path.exists():
-        raise ValueError(f"Service path does not exist: {service_path}")
-
-    with context.cd(service_path):
-        context.run("docker-compose restart")
 
 
-services = invoke.Collection("services")
+services = invoke.Collection("service")
 services.add_task(start)
 services.add_task(stop)
-services.add_task(restart)
 
 server = invoke.Collection("server")
 
@@ -100,3 +157,4 @@ server.add_task(system_reboot, name="reboot")
 
 ns.add_collection(server)
 ns.add_collection(services)
+ns.add_task(generate_configuration)