From 57a2b02c746e7f3c55e57ea973deb53d85d7c45d Mon Sep 17 00:00:00 2001 From: Marc Cataford Date: Mon, 4 Sep 2023 14:22:53 -0400 Subject: [PATCH] infra: remove docker compose dependency (#28) * infra(techdebt): migrate bastion, deluge to not use compose * infra: migrate plex, move env files out of repository path * infra: migrate bitwarden * infra: add version tagging * chore: linting + docs --- .gitignore | 1 + requirements.in | 2 + requirements.txt | 6 +- services.yml.j2 | 48 ++++++++++ services/Dockerfile-bastion | 7 ++ services/Dockerfile-bitwarden | 1 + services/Dockerfile-bitwarden-db | 1 + services/Dockerfile-deluge | 5 + services/Dockerfile-plex | 3 + tasks.py | 158 +++++++++++++++++++++---------- 10 files changed, 181 insertions(+), 51 deletions(-) create mode 100644 services.yml.j2 create mode 100644 services/Dockerfile-bastion create mode 100644 services/Dockerfile-bitwarden create mode 100644 services/Dockerfile-bitwarden-db create mode 100644 services/Dockerfile-deluge create mode 100644 services/Dockerfile-plex diff --git a/.gitignore b/.gitignore index 266f8cf..7388990 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ spadinaistan.venv +services.yml **/*.env **/.env pyinfra-debug.log diff --git a/requirements.in b/requirements.in index 515cac5..8f3058e 100644 --- a/requirements.in +++ b/requirements.in @@ -1,3 +1,5 @@ black invoke pyinfra +pyyaml ~= 6.0.0 +jinja2 ~= 3.1.0 diff --git a/requirements.txt b/requirements.txt index dac7971..b125d97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,9 @@ idna==3.4 invoke==2.1.0 # via -r ./requirements.in jinja2==3.1.2 - # via pyinfra + # via + # -r ./requirements.in + # pyinfra markupsafe==2.1.2 # via jinja2 mypy-extensions==1.0.0 @@ -65,6 +67,8 @@ python-dateutil==2.8.2 # via pyinfra pywinrm==0.4.3 # via pyinfra +pyyaml==6.0.1 + # via -r ./requirements.in requests==2.29.0 # via # pywinrm diff --git a/services.yml.j2 b/services.yml.j2 new file mode 100644 index 0000000..9c5a6a4 --- /dev/null +++ b/services.yml.j2 @@ -0,0 +1,48 @@ +services: + deluge: + ports: + - 8112:8112 + - 6881:6881/udp + - 6881:6881/tcp + volumes: + - {{ DELUGE_CONFIG_ROOT }}:/config + - {{ DELUGE_DOWNLOADS_ROOT }}:/downloads + - {{ DELUGE_COMPLETE_DOWNLOADS_ROOT }}:/complete + env_files: + - {{ ENVS_ROOT }}/deluge.env + bastion: + env_files: + - {{ ENVS_ROOT }}/bastion.env + bitwarden: + env_files: + - {{ ENVS_ROOT }}/bitwarden-web.env + port: + - 7000:8080 + - 7001:8443 + volumes: + - {{ BITWARDEN_CONFIG_ROOT }}:/etc/bitwarden + - {{ BITWARDEN_LOGS_ROOT }}:/var/log/bitwarden + bitwarden-db: + env_files: + - {{ ENVS_ROOT }}/bitwarden-db.env + volumes: + - {{ BITWARDEN_DB_ROOT }}:/var/lib/mysql + ports: + - 3306:3306 + plex: + ports: + - 32400:32400/tcp + - 32469:32469/tcp + - 3005:3005/tcp + - 8324:8324/tcp + - 1900:1900/udp + - 32410:32410/udp + - 32412:32412/udp + - 32413:32413/udp + - 32414:32414/udp + env_files: + - {{ ENVS_ROOT }}/plex.env + volumes: + - {{ PLEX_DB_ROOT }}:/config + - {{ PLEX_TRANSCODE_ROOT }}:/transcode + - {{ PLEX_DATA_ROOT }}:/data diff --git a/services/Dockerfile-bastion b/services/Dockerfile-bastion new file mode 100644 index 0000000..f08154f --- /dev/null +++ b/services/Dockerfile-bastion @@ -0,0 +1,7 @@ +FROM cloudflare/cloudflared:1414-cb4bd8d06572 + +ARG TUNNEL_SECRET + +ENV TUNNEL_TOKEN=$TUNNEL_SECRET + +CMD ["tunnel", "run"] diff --git a/services/Dockerfile-bitwarden b/services/Dockerfile-bitwarden new file mode 100644 index 0000000..4371da0 --- /dev/null +++ b/services/Dockerfile-bitwarden @@ -0,0 +1 @@ +FROM bitwarden/self-host:beta diff --git a/services/Dockerfile-bitwarden-db b/services/Dockerfile-bitwarden-db new file mode 100644 index 0000000..20240a0 --- /dev/null +++ b/services/Dockerfile-bitwarden-db @@ -0,0 +1 @@ +FROM mariadb:10 diff --git a/services/Dockerfile-deluge b/services/Dockerfile-deluge new file mode 100644 index 0000000..bb90119 --- /dev/null +++ b/services/Dockerfile-deluge @@ -0,0 +1,5 @@ +FROM lscr.io/linuxserver/deluge:latest + +ENV TZ=America/Toronto + +EXPOSE 8112 6881/tcp 6881/udp diff --git a/services/Dockerfile-plex b/services/Dockerfile-plex new file mode 100644 index 0000000..467cf25 --- /dev/null +++ b/services/Dockerfile-plex @@ -0,0 +1,3 @@ +FROM plexinc/pms-docker:1.32.0.6973-a787c5a8e + +ENV TZ=America/Toronto diff --git a/tasks.py b/tasks.py index 465f101..970b369 100644 --- a/tasks.py +++ b/tasks.py @@ -1,5 +1,8 @@ import invoke +import jinja2 +import yaml +import os import pathlib import typing @@ -8,6 +11,42 @@ ns = invoke.Collection() PYINFRA_COMMON_PREFIX = "pyinfra -vvv pyinfra/inventory.py" +def _get_tag(ctx: invoke.context.Context) -> str: + """ + Gets the current tag, if it exists, for image versioning purposes. + """ + try: + out = ctx.run("git describe --tags --exact-match", hide=True) + return out.stdout.strip() + except: + return "dev" + + +def _ensure_networks(ctx: invoke.context.Context): + """ + Ensures that the required networks exist. + """ + print("Ensuring networks exist...") + output = ctx.run("docker network ls -f name=spad-internal -q", hide=True) + if not bool(output.stdout): + ctx.run("docker network create --scope=swarm spad-internal") + print('\t✅ Created network "internal".') + else: + print('\t✅ Network "internal" already exists.') + + +def _ensure_swarm(ctx: invoke.context.Context): + """ + Ensures that swarm mode is on. + """ + print("Ensuring swarm mode is on...") + try: + ctx.run("docker swarm init --advertise-addr 192.168.1.17", hide=True) + except: + pass + print("\t✅ Swarm in on.") + + @invoke.task() def system_updates(ctx): ctx.run(f"{PYINFRA_COMMON_PREFIX} pyinfra/system_updates.py") @@ -19,79 +58,97 @@ def system_reboot(ctx): @invoke.task() -def start(context: invoke.context.Context, service: typing.Optional[str]): +def generate_configuration(ctx): """ - Starts a service. + Generates the service configuration file `services.yml` + based on environment variables. - Services are assumed to be docker-compose-friendly and start as - docker-compose up -d. - - The supplied service name is used to pathing, with the expected - file structure being - / - /services - /service1 - docker-compose.yml + To avoid polluting the environment, this can be called as + `env $(cat .env | xargs) inv generate-configuration` where + .env contains the variables. """ + template_loader = jinja2.FileSystemLoader(searchpath="./") + template_env = jinja2.Environment( + loader=template_loader, undefined=jinja2.StrictUndefined + ) + template = template_env.get_template("services.yml.j2") - if service is None: - raise ValueError("Service name must be provided.") - - service_path = pathlib.Path("services", service) - - if not service_path.exists(): - raise ValueError(f"Service path does not exist: {service_path}") - - with context.cd(service_path): - context.run("docker-compose up --build --force-recreate -d") + with open("services.yml", "w") as outfile: + outfile.write(template.render(**os.environ)) @invoke.task() -def stop(context: invoke.context.Context, service: typing.Optional[str]): +def start(ctx, services: str): """ - Stops a service. + Starts one of more services, defined by a comma-separated list of labels. - The same assumptions about file and service structure as made as with - `start `. + Services should correspond to an entry in `services.yml`. """ + _ensure_swarm(ctx) + _ensure_networks(ctx) - if service is None: - raise ValueError("Service name must be provided.") + current_version = _get_tag(ctx) - service_path = pathlib.Path("services", service) + print("Starting services...") - if not service_path.exists(): - raise ValueError(f"Service path does not exist: {service_path}") + for service in services.split(","): + with open("services.yml", "r") as config: + service_config = yaml.load(config, Loader=yaml.Loader)["services"][service] - with context.cd(service_path): - context.run("docker-compose down") + ctx.run( + f"docker build -t spadinaistan-{service}:{current_version} -f services/Dockerfile-{service} .", + hide=True, + ) + ports_args = ( + f"--publish {ports}\\\n" for ports in service_config.get("ports", []) + ) + + volume_args = [] + + for volume in service_config.get("volumes", []): + source, target = volume.split(":") + volume_args.append(f"--mount type=bind,source={source},target={target}\\\n") + + env_files = ( + f"--env-file {env_file}\\\n" + for env_file in service_config.get("env_files", []) + ) + + try: + out = ctx.run( + f"""docker service create \ + --name spad-{service}\ + --network spad-internal\ + {" ".join(env_files)}\ + --hostname=\"{service}\"\ + {" ".join(volume_args)}\ + {" ".join(ports_args)}\ + spadinaistan-{service}:{current_version}""", + hide=True, + ) + except: + print(out.stdout) + print(out.stderr) + continue + + print(f"\t✅ {service} is running.") @invoke.task() -def restart(context: invoke.context.Context, service: typing.Optional[str]): +def stop(ctx: invoke.context.Context, services: str): """ - Restarts a service. - - The same assumptions about file and service structure as made as with - `start `. + Stops the provided list of services, as a comma-separated list + of labels. """ - - if service is None: - raise ValueError("Service name must be provided.") - - service_path = pathlib.Path("services", service) - - if not service_path.exists(): - raise ValueError(f"Service path does not exist: {service_path}") - - with context.cd(service_path): - context.run("docker-compose restart") + print("Stopping services...") + for service in services.split(","): + ctx.run(f"docker service rm spad-{service}", hide=True) + print(f"\t✅ {service} is stopped.") -services = invoke.Collection("services") +services = invoke.Collection("service") services.add_task(start) services.add_task(stop) -services.add_task(restart) server = invoke.Collection("server") @@ -100,3 +157,4 @@ server.add_task(system_reboot, name="reboot") ns.add_collection(server) ns.add_collection(services) +ns.add_task(generate_configuration)