diff --git a/README.md b/README.md index 3a7b6ef..0452ce6 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ anyone else. ## Getting started -Use `. script/bootstrap` to set up the Python environment needed for the invoke and pyinfra tooling to work. +Use `. bootstrap.sh` to set up Task. -This expects `pyenv` to be set up on your system. +Once this is done, `task -l` will outline all available commands. ## Configuration @@ -30,6 +30,8 @@ env: ENV_FILE_DIR: ... # Name of the network associated with the tunnel exposing services. SHARED_NETWORK_NAME: ... + # Path to the storage root. + STORAGE_DIR: ... ``` The `env.yml` file is ignored by version-control. diff --git a/script/bootstrap-tasks b/bootstrap.sh similarity index 100% rename from script/bootstrap-tasks rename to bootstrap.sh diff --git a/pyinfra/__init__.py b/pyinfra/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pyinfra/inventory.py b/pyinfra/inventory.py deleted file mode 100644 index d910c67..0000000 --- a/pyinfra/inventory.py +++ /dev/null @@ -1 +0,0 @@ -hosts = [("spadinaistan.karnov.club", {"ssh_port": 6969})] diff --git a/pyinfra/reboot.py b/pyinfra/reboot.py deleted file mode 100644 index 65031a9..0000000 --- a/pyinfra/reboot.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Reboot machines. -""" - -import pyinfra - - -def reboot_machines(): - pyinfra.operations.server.reboot( - name="Reboot server", _sudo=True, _use_sudo_password=True - ) - - -reboot_machines() diff --git a/pyinfra/system_updates.py b/pyinfra/system_updates.py deleted file mode 100644 index 9f217ef..0000000 --- a/pyinfra/system_updates.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Runs Ubuntu system updates. -""" - -import pyinfra - - -def update_ubuntu_packages(): - common = {"_sudo": True, "_use_sudo_password": True} - pyinfra.operations.server.shell( - name="Search for system updates", commands=["apt update"], **common - ) - pyinfra.operations.server.shell( - name="Apply updates and remove orphaned packages", - commands=["apt upgrade --autoremove -y"], - **common - ) - - -update_ubuntu_packages() diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 8f3058e..0000000 --- a/requirements.in +++ /dev/null @@ -1,5 +0,0 @@ -black -invoke -pyinfra -pyyaml ~= 6.0.0 -jinja2 ~= 3.1.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b125d97..0000000 --- a/requirements.txt +++ /dev/null @@ -1,93 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile ./requirements.in -# -bcrypt==4.0.1 - # via paramiko -black==23.3.0 - # via -r ./requirements.in -certifi==2022.12.7 - # via requests -cffi==1.15.1 - # via - # cryptography - # pynacl -charset-normalizer==3.1.0 - # via requests -click==8.1.3 - # via - # black - # pyinfra -colorama==0.4.6 - # via pyinfra -configparser==5.3.0 - # via pyinfra -cryptography==40.0.2 - # via - # paramiko - # pyspnego - # requests-ntlm -distro==1.8.0 - # via pyinfra -gevent==22.10.2 - # via pyinfra -greenlet==2.0.2 - # via gevent -idna==3.4 - # via requests -invoke==2.1.0 - # via -r ./requirements.in -jinja2==3.1.2 - # via - # -r ./requirements.in - # pyinfra -markupsafe==2.1.2 - # via jinja2 -mypy-extensions==1.0.0 - # via black -packaging==23.1 - # via black -paramiko==2.12.0 - # via pyinfra -pathspec==0.11.1 - # via black -platformdirs==3.5.0 - # via black -pycparser==2.21 - # via cffi -pyinfra==2.6.2 - # via -r ./requirements.in -pynacl==1.5.0 - # via paramiko -pyspnego==0.9.0 - # via requests-ntlm -python-dateutil==2.8.2 - # via pyinfra -pywinrm==0.4.3 - # via pyinfra -pyyaml==6.0.1 - # via -r ./requirements.in -requests==2.29.0 - # via - # pywinrm - # requests-ntlm -requests-ntlm==1.2.0 - # via pywinrm -six==1.16.0 - # via - # paramiko - # python-dateutil - # pywinrm -urllib3==1.26.15 - # via requests -xmltodict==0.13.0 - # via pywinrm -zope-event==4.6 - # via gevent -zope-interface==6.0 - # via gevent - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/script/bootstrap b/script/bootstrap deleted file mode 100644 index 6e67930..0000000 --- a/script/bootstrap +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -VENV="spadinaistan.venv" - -python -m pip install pip~=23.1 pip-tools==6.13.0 --no-cache - -if [ ! -d "./$VENV" ]; then - python -m venv ./$VENV -fi - -source ./$VENV/bin/activate - - -pip-sync ./requirements.txt diff --git a/script/lock b/script/lock deleted file mode 100644 index 9e6b834..0000000 --- a/script/lock +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -pip-compile ./requirements.in - diff --git a/services.yml.j2 b/services.yml.j2 deleted file mode 100644 index 209de4f..0000000 --- a/services.yml.j2 +++ /dev/null @@ -1,47 +0,0 @@ -services: - deluge: - ports: - - 8112:8112 - - 6881:6881/udp - - 6881:6881/tcp - volumes: - - {{ DELUGE_CONFIG_ROOT }}:/config - - {{ DELUGE_DOWNLOADS_ROOT }}:/downloads - - {{ DELUGE_COMPLETE_DOWNLOADS_ROOT }}:/complete - env_files: - - {{ ENVS_ROOT }}/deluge.env - bastion: - env_files: - - {{ ENVS_ROOT }}/bastion.env - bitwarden: - env_files: - - {{ ENVS_ROOT }}/bitwarden-web.env - ports: - - 8080:8080 - volumes: - - {{ BITWARDEN_CONFIG_ROOT }}:/etc/bitwarden - - {{ BITWARDEN_LOGS_ROOT }}:/var/log/bitwarden - bitwarden-db: - env_files: - - {{ ENVS_ROOT }}/bitwarden-db.env - volumes: - - {{ BITWARDEN_DB_ROOT }}:/var/lib/mysql - ports: - - 3306:3306 - plex: - ports: - - published=32400,target=32400,mode=host,protocol=tcp - - 32469:32469/tcp - - 3005:3005/tcp - - 8324:8324/tcp - - 1900:1900/udp - - 32410:32410/udp - - 32412:32412/udp - - 32413:32413/udp - - 32414:32414/udp - env_files: - - {{ ENVS_ROOT }}/plex.env - volumes: - - {{ PLEX_DB_ROOT }}:/config - - {{ PLEX_TRANSCODE_ROOT }}:/transcode - - {{ PLEX_DATA_ROOT }}:/data diff --git a/services/Dockerfile-bastion b/services/bastion/Dockerfile-bastion-app similarity index 100% rename from services/Dockerfile-bastion rename to services/bastion/Dockerfile-bastion-app diff --git a/services/bastion/build.sh b/services/bastion/build.sh new file mode 100644 index 0000000..c1c8dbc --- /dev/null +++ b/services/bastion/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source constants.sh + +docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-bastion-app . diff --git a/services/bastion/constants.sh b/services/bastion/constants.sh new file mode 100644 index 0000000..6a11f1c --- /dev/null +++ b/services/bastion/constants.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +export APP_NAME="bastion" +export APP_CONTAINER_NAME=$APP_NAME-app +export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME +export NETWORK_NAME=$APP_NAME-local diff --git a/services/bastion/start.sh b/services/bastion/start.sh new file mode 100644 index 0000000..9127429 --- /dev/null +++ b/services/bastion/start.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +source constants.sh + +docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists" + +docker run \ + --detach \ + --network $NETWORK_NAME \ + --name $APP_CONTAINER_NAME \ + --env-file $ENV_FILE_DIR/bastion.env \ + $APP_IMAGE_NAME:dev diff --git a/services/bastion/stop.sh b/services/bastion/stop.sh new file mode 100644 index 0000000..871ff06 --- /dev/null +++ b/services/bastion/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source constants.sh + +docker rm -f $APP_CONTAINER_NAME diff --git a/services/Dockerfile-bitwarden b/services/bitwarden/Dockerfile-bitwarden-app similarity index 100% rename from services/Dockerfile-bitwarden rename to services/bitwarden/Dockerfile-bitwarden-app diff --git a/services/Dockerfile-bitwarden-db b/services/bitwarden/Dockerfile-bitwarden-db similarity index 100% rename from services/Dockerfile-bitwarden-db rename to services/bitwarden/Dockerfile-bitwarden-db diff --git a/services/bitwarden/build.sh b/services/bitwarden/build.sh new file mode 100644 index 0000000..59dd88f --- /dev/null +++ b/services/bitwarden/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source constants.sh + +docker build -t $DB_IMAGE_NAME:dev -f Dockerfile-bitwarden-db . +docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-bitwarden-app . diff --git a/services/bitwarden/constants.sh b/services/bitwarden/constants.sh new file mode 100644 index 0000000..2df6d9c --- /dev/null +++ b/services/bitwarden/constants.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +export APP_NAME="bitwarden" +export APP_CONTAINER_NAME=$APP_NAME-app +export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME +export DB_CONTAINER_NAME=$APP_NAME-db +export DB_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$DB_CONTAINER_NAME +export NETWORK_NAME=$APP_NAME-local diff --git a/services/bitwarden/start.sh b/services/bitwarden/start.sh new file mode 100644 index 0000000..a281880 --- /dev/null +++ b/services/bitwarden/start.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +source constants.sh + +docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists" + +docker run \ + --detach \ + --network $NETWORK_NAME \ + --name $DB_CONTAINER_NAME \ + --env-file $ENV_FILE_DIR/bitwarden-db.env \ + --mount type=bind,source=$APP_DATA_DIR/bitwarden/db,target=/var/lib/mysql \ + $DB_IMAGE_NAME:dev + +docker run \ + --detach \ + --network $NETWORK_NAME \ + --name $APP_CONTAINER_NAME \ + --env-file $ENV_FILE_DIR/bitwarden-web.env \ + --publish 8080:8080 \ + --mount type=bind,source=$APP_DATA_DIR/bitwarden/config,target=/etc/bitwarden \ + --mount type=bind,source=$APP_DATA_DIR/bitwarden/logs,target=/var/log/bitwarden \ + $APP_IMAGE_NAME:dev diff --git a/services/bitwarden/stop.sh b/services/bitwarden/stop.sh new file mode 100644 index 0000000..62c5c50 --- /dev/null +++ b/services/bitwarden/stop.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source constants.sh + +docker rm -f $APP_CONTAINER_NAME +docker rm -f $DB_CONTAINER_NAME diff --git a/services/Dockerfile-deluge b/services/deluge/Dockerfile-deluge-app similarity index 100% rename from services/Dockerfile-deluge rename to services/deluge/Dockerfile-deluge-app diff --git a/services/deluge/build.sh b/services/deluge/build.sh new file mode 100644 index 0000000..d2adb1b --- /dev/null +++ b/services/deluge/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source constants.sh + +docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-deluge-app . diff --git a/services/deluge/constants.sh b/services/deluge/constants.sh new file mode 100644 index 0000000..e68981f --- /dev/null +++ b/services/deluge/constants.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +export APP_NAME="deluge" +export APP_CONTAINER_NAME=$APP_NAME-app +export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME +export NETWORK_NAME=$APP_NAME-local diff --git a/services/deluge/start.sh b/services/deluge/start.sh new file mode 100644 index 0000000..69b05e6 --- /dev/null +++ b/services/deluge/start.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source constants.sh + +docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists" + +docker run \ + --detach \ + --network $NETWORK_NAME \ + --name $APP_CONTAINER_NAME \ + --env-file $ENV_FILE_DIR/deluge.env \ + --publish 8112:8112 \ + --publish 6881:6881/udp \ + --publish 6881:6881/tcp \ + --mount type=bind,source=$APP_DATA_DIR/deluge/config,target=/config \ + --mount type=bind,source=$APP_DATA_DIR/deluge/downloads,target=/downloads \ + --mount type=bind,source=$STORAGE_DIR,target=/complete \ + $APP_IMAGE_NAME:dev diff --git a/services/deluge/stop.sh b/services/deluge/stop.sh new file mode 100644 index 0000000..871ff06 --- /dev/null +++ b/services/deluge/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source constants.sh + +docker rm -f $APP_CONTAINER_NAME diff --git a/services/Dockerfile-plex b/services/plex/Dockerfile-plex-app similarity index 100% rename from services/Dockerfile-plex rename to services/plex/Dockerfile-plex-app diff --git a/services/plex/build.sh b/services/plex/build.sh new file mode 100644 index 0000000..27a376f --- /dev/null +++ b/services/plex/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source constants.sh + +docker build -t $APP_IMAGE_NAME:dev -f Dockerfile-plex-app . diff --git a/services/plex/constants.sh b/services/plex/constants.sh new file mode 100644 index 0000000..f4c5085 --- /dev/null +++ b/services/plex/constants.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +export APP_NAME="plex" +export APP_CONTAINER_NAME=$APP_NAME-app +export APP_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$APP_CONTAINER_NAME +export DB_CONTAINER_NAME=$APP_NAME-db +export DB_IMAGE_NAME=$CONTAINER_NAME_PREFIX-$DB_CONTAINER_NAME +export NETWORK_NAME=$APP_NAME-local diff --git a/services/plex/start.sh b/services/plex/start.sh new file mode 100644 index 0000000..f84ee09 --- /dev/null +++ b/services/plex/start.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +source constants.sh + +docker network create $NETWORK_NAME || echo "Network $NETWORK_NAME already exists" + +docker run \ + --detach \ + --network $NETWORK_NAME \ + --name $APP_CONTAINER_NAME \ + --env-file $ENV_FILE_DIR/plex.env \ + --publish 32401:32400 \ + --publish 32469:32469 \ + --publish 3005:3005 \ + --publish 8324:8324 \ + --publish 1900:1900/udp \ + --publish 32410:32410/udp \ + --publish 32412:32412/udp \ + --publish 32413:32413/udp \ + --publish 32414:32414/udp \ + --mount type=bind,source=$APP_DATA_DIR/plex/database,target=/config \ + --mount type=bind,source=$APP_DATA_DIR/plex/transcode,target=/transcode \ + --mount type=bind,source=$STORAGE_DIR/media,target=/data \ + $APP_IMAGE_NAME:dev diff --git a/services/plex/stop.sh b/services/plex/stop.sh new file mode 100644 index 0000000..871ff06 --- /dev/null +++ b/services/plex/stop.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source constants.sh + +docker rm -f $APP_CONTAINER_NAME diff --git a/tasks.py b/tasks.py deleted file mode 100644 index 970b369..0000000 --- a/tasks.py +++ /dev/null @@ -1,160 +0,0 @@ -import invoke -import jinja2 -import yaml - -import os -import pathlib -import typing - -ns = invoke.Collection() - -PYINFRA_COMMON_PREFIX = "pyinfra -vvv pyinfra/inventory.py" - - -def _get_tag(ctx: invoke.context.Context) -> str: - """ - Gets the current tag, if it exists, for image versioning purposes. - """ - try: - out = ctx.run("git describe --tags --exact-match", hide=True) - return out.stdout.strip() - except: - return "dev" - - -def _ensure_networks(ctx: invoke.context.Context): - """ - Ensures that the required networks exist. - """ - print("Ensuring networks exist...") - output = ctx.run("docker network ls -f name=spad-internal -q", hide=True) - if not bool(output.stdout): - ctx.run("docker network create --scope=swarm spad-internal") - print('\t✅ Created network "internal".') - else: - print('\t✅ Network "internal" already exists.') - - -def _ensure_swarm(ctx: invoke.context.Context): - """ - Ensures that swarm mode is on. - """ - print("Ensuring swarm mode is on...") - try: - ctx.run("docker swarm init --advertise-addr 192.168.1.17", hide=True) - except: - pass - print("\t✅ Swarm in on.") - - -@invoke.task() -def system_updates(ctx): - ctx.run(f"{PYINFRA_COMMON_PREFIX} pyinfra/system_updates.py") - - -@invoke.task() -def system_reboot(ctx): - ctx.run(f"{PYINFRA_COMMON_PREFIX} pyinfra/reboot.py") - - -@invoke.task() -def generate_configuration(ctx): - """ - Generates the service configuration file `services.yml` - based on environment variables. - - To avoid polluting the environment, this can be called as - `env $(cat .env | xargs) inv generate-configuration` where - .env contains the variables. - """ - template_loader = jinja2.FileSystemLoader(searchpath="./") - template_env = jinja2.Environment( - loader=template_loader, undefined=jinja2.StrictUndefined - ) - template = template_env.get_template("services.yml.j2") - - with open("services.yml", "w") as outfile: - outfile.write(template.render(**os.environ)) - - -@invoke.task() -def start(ctx, services: str): - """ - Starts one of more services, defined by a comma-separated list of labels. - - Services should correspond to an entry in `services.yml`. - """ - _ensure_swarm(ctx) - _ensure_networks(ctx) - - current_version = _get_tag(ctx) - - print("Starting services...") - - for service in services.split(","): - with open("services.yml", "r") as config: - service_config = yaml.load(config, Loader=yaml.Loader)["services"][service] - - ctx.run( - f"docker build -t spadinaistan-{service}:{current_version} -f services/Dockerfile-{service} .", - hide=True, - ) - ports_args = ( - f"--publish {ports}\\\n" for ports in service_config.get("ports", []) - ) - - volume_args = [] - - for volume in service_config.get("volumes", []): - source, target = volume.split(":") - volume_args.append(f"--mount type=bind,source={source},target={target}\\\n") - - env_files = ( - f"--env-file {env_file}\\\n" - for env_file in service_config.get("env_files", []) - ) - - try: - out = ctx.run( - f"""docker service create \ - --name spad-{service}\ - --network spad-internal\ - {" ".join(env_files)}\ - --hostname=\"{service}\"\ - {" ".join(volume_args)}\ - {" ".join(ports_args)}\ - spadinaistan-{service}:{current_version}""", - hide=True, - ) - except: - print(out.stdout) - print(out.stderr) - continue - - print(f"\t✅ {service} is running.") - - -@invoke.task() -def stop(ctx: invoke.context.Context, services: str): - """ - Stops the provided list of services, as a comma-separated list - of labels. - """ - print("Stopping services...") - for service in services.split(","): - ctx.run(f"docker service rm spad-{service}", hide=True) - print(f"\t✅ {service} is stopped.") - - -services = invoke.Collection("service") -services.add_task(start) -services.add_task(stop) - -server = invoke.Collection("server") - -server.add_task(system_updates, name="update") -server.add_task(system_reboot, name="reboot") - -ns.add_collection(server) -ns.add_collection(services) -ns.add_task(generate_configuration)