From 9a81426840e347c2b7b1bdddb3d931fff62a8e3e Mon Sep 17 00:00:00 2001 From: Marc Cataford Date: Fri, 19 Mar 2021 13:38:54 -0400 Subject: [PATCH] Migrate to Terraform (#3) * infra: exit sceptre, enter terraform infra: ignores, vars * infra: tweaks * ci: remove sceptre validation * ci: bootstrap * build: deps * ci: py setup * docs: add docs to tasks * docs: update --- .github/workflows/base.yml | 17 +- .gitignore | 4 + README.md | 19 +- infrastructure/app/app.tf | 82 +++++++ infrastructure/app/variables.tf | 11 + infrastructure/bootstrap/outputs.tf | 3 + infrastructure/bootstrap/provider.tf | 5 + infrastructure/bootstrap/resources.tf | 8 + infrastructure/bootstrap/variables.tf | 7 + infrastructure/config/app/app.yaml | 8 - infrastructure/config/app/config.yaml | 2 - .../config/bootstrap/bootstrap.yaml | 4 - infrastructure/config/bootstrap/config.yaml | 2 - infrastructure/config/config.yaml | 2 - infrastructure/templates/app.yaml | 99 -------- infrastructure/templates/bootstrap.yaml | 18 -- requirements.in | 0 requirements.txt | 6 + requirements_dev.in | 6 + requirements_dev.txt | 203 +++++++++++++++-- script/bootstrap | 38 ++-- tasks.py | 213 ++++++++++++++---- 22 files changed, 518 insertions(+), 239 deletions(-) create mode 100644 infrastructure/app/app.tf create mode 100644 infrastructure/app/variables.tf create mode 100644 infrastructure/bootstrap/outputs.tf create mode 100644 infrastructure/bootstrap/provider.tf create mode 100644 infrastructure/bootstrap/resources.tf create mode 100644 infrastructure/bootstrap/variables.tf delete mode 100644 infrastructure/config/app/app.yaml delete mode 100644 infrastructure/config/app/config.yaml delete mode 100644 infrastructure/config/bootstrap/bootstrap.yaml delete mode 100644 infrastructure/config/bootstrap/config.yaml delete mode 100644 infrastructure/config/config.yaml delete mode 100644 infrastructure/templates/app.yaml delete mode 100644 infrastructure/templates/bootstrap.yaml create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 requirements_dev.in diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index ae6a751..d2c1984 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -6,17 +6,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.8.2' - name: Install dependencies - run: | - sudo apt install python3-venv - python3 -m pip install --upgrade pip setuptools wheel pipx - - name: Validate templates - run: | - cd infrastructure - python3 -m pipx run sceptre validate app/app.yaml - python3 -m pipx run sceptre validate bootstrap/bootstrap.yaml - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }} + run: CI=1 . script/bootstrap - name: Lint - run: python3 -m pipx run black . --check + run: inv lint diff --git a/.gitignore b/.gitignore index 0de1eff..dc9b11d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ *.pyc *.zip *_out.json +*.terraform* +*tfstate* +*tfvars + diff --git a/README.md b/README.md index 8793a80..2cf387c 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,15 @@ ## Overview -AWS Lambdas are fun, but often the amount of boilerplate involved in getting a project off the ground hinders the fun. From setting up a local environment to writing out a Cloudformation template, the overhead of Lambda-based greenfield projects can be daunting. No more. Just use this repository as a template or clone it and jump straight into the action! This repository offers a quick template your can use, full with a Docker setup for local development and invocation commands that you can use to package and deploy small Lambdas. - -Use this as a foundation and tweak it to your use case! +AWS Lambdas are fun, but often the amount of boilerplate involved in getting a project off the ground hinders the fun. From setting up a local environment to writing out infrastructure templates, the overhead of Lambda-based greenfield projects can be daunting. No more. Just use this repository as a template or clone it and jump straight into the action! This repository offers a quick template your can use, full with a Docker setup for local development and invocation commands that you can use to package and deploy Lambda-based apps. ## Local development To get started, hit the bootstrap script with `. script/bootstrap`. This will set up a Python 3.8 virtualenv set up with some basic tools that will make your life easier. -The base Lambda handler is at `src/base.py` and all the infrastructure templates and Sceptre configuration are in `infrastructure`. +The base Lambda handler is at `src/base.py` and all the Terraform configurations are in `infrastructure`. -[Read more about Sceptre](https://sceptre.cloudreach.com/latest/index.html) +[Read more about Sceptre](https://sceptre.cloudreach.com/latest/index.html://www.terraform.io/docs/index.html) [Read more about AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html) @@ -23,18 +21,11 @@ The base Lambda handler is at `src/base.py` and all the infrastructure templates This template uses PyInvoke, all commands are of the format `inv `. -|Command|Description| -|---|---| -|`app.start`|Start your Lambda in Docker.| -|`app.stop`|Stop and remove the container.| -|`app.invoke-function `|Invokes the given local Lambda by container name| -|`stack.deploy`|Packages your code and deploys the stack| -|`stack.teardown-app`|Tears down the application stack| -|`stack.teardown-bootstrap`|Tears down the bootstrap stack| +Use `inv --list` for the full list of commands. ## Deployment -The base setup assumes that your Lambda handler is located in `src.base`. Doing `inv stack.deploy` will zip up your `src` directory, create an S3 bucket for your development artifacts, uploads the source doe archive to S3 and kickstarts the Cloudformation deployment of your stack. +Deployment is in three steps: on first setup, you will need to make sure that your `bootstrap` environment is ready via `inv cloud.apply bootstrap`. Then, you should upload your lambdas' source with `inv cloud.pack` and `inv cloud.push`. Finally, you can deploy your application resources with `inv cloud.deploy app`. ## Contributing diff --git a/infrastructure/app/app.tf b/infrastructure/app/app.tf new file mode 100644 index 0000000..6733917 --- /dev/null +++ b/infrastructure/app/app.tf @@ -0,0 +1,82 @@ +provider aws { + profile = "default" + region = var.aws_region +} + +resource "aws_iam_role" "lambda_role" { + name = "lambda_role" + assume_role_policy = < str: + return str(Path(BASE_PATH, path).absolute()) + + +def _build_help_dict(segments: List[str]) -> Dict[str, str]: + return {segment: HELP_SEGMENTS[segment] for segment in segments} + + +PROJECT_PATHS = { + "app": _compose_path("infrastructure/app"), + "bootstrap": _compose_path("infrastructure/bootstrap"), +} ##################### -# Stack invocations # +# Cloud invocations # ##################### -@task(name="teardown-app") -def teardown_app(ctx): - with ctx.cd("infrastructure"): - ctx.run("sceptre delete app/app.yaml -y") +@task(name="plan", help=_build_help_dict(["project"])) +def cloud_plan(ctx, project): + """ + Builds the Terraform plan for the given project. + """ + with ctx.cd(PROJECT_PATHS[project]): + ctx.run(f"terraform plan --var-file {VARIABLES_PATH}") -@task(name="teardown-bootstrap") -def teardown_bootstrap(ctx): - with ctx.cd("infrastructure"): - ctx.run("sceptre delete bootstrap/bootstrap.yaml -y") +@task(name="apply", help=_build_help_dict(["project"])) +def cloud_apply(ctx, project): + """ + Applies infrastructure changes to the given project. + """ + with ctx.cd(PROJECT_PATHS[project]): + ctx.run("terraform taint --allow-missing aws_lambda_function.apgnd_lambda_func") + ctx.run("terraform taint --allow-missing aws_lambda_permission.apigw") + ctx.run(f"terraform apply --var-file {VARIABLES_PATH}") -@task(name="deploy") -def stack_deploy(ctx): - path = Path(__file__).parent - with ctx.cd("infrastructure"): - ctx.run("sceptre launch bootstrap/bootstrap.yaml -y") +@task(name="destroy", help=_build_help_dict(["project"])) +def cloud_destroy(ctx, project): + """ + Destroys resources associated with the given project. + """ + with ctx.cd(PROJECT_PATHS[project]): + ctx.run(f"terraform destroy --var-file {VARIABLES_PATH}") - ctx.run("zip lambda_function.zip src/*") - with open("lambda_function.zip", "rb") as src: - srchash = hashlib.md5(src.read()).hexdigest() - new_archive_name = f"lambda_function_{srchash}.zip" - ctx.run(f"mv lambda_function.zip {new_archive_name}") - ctx.run( - f"aws s3 cp {new_archive_name} s3://mcat-dev-test-bucket-artifacts-2 && rm {new_archive_name}" - ) +@task(name="pack") +def cloud_pack(ctx): + """ + Prepares and packages the source code for lambdas. + """ + with ctx.cd(BASE_PATH): + ctx.run("pip install -r requirements.txt --target package/") + ctx.run("zip -r lambda_function.zip src/*") - with ctx.cd("infrastructure"): - ctx.run(f"sceptre --var source_key={new_archive_name} launch app/app.yaml -y") + with ctx.cd(_compose_path("package")): + ctx.run("zip -r ../lambda_function.zip ./") + + +@task(name="push", help=_build_help_dict(["archive"])) +def cloud_push(ctx, archive): + """ + Pushes the given archive to S3. + """ + artifacts_bucket = None + + with ctx.cd(_compose_path(PROJECT_PATHS["bootstrap"])): + out = ctx.run("terraform output", hide="out").stdout + artifacts_bucket_match = re.match( + "artifacts_bucket_name = (?P[0-9a-zA-Z\-]+)\n", out + ) + artifacts_bucket = artifacts_bucket_match.group("bucket_name") + + with ctx.cd(BASE_PATH): + ctx.run(f"aws s3 cp {archive} s3://{artifacts_bucket}", hide="out") + + print(f"Uploaded {archive} to s3 ({artifacts_bucket})!") ##################### @@ -47,43 +106,109 @@ def stack_deploy(ctx): @task(name="start") -def app_start(ctx): +def local_start(ctx): + """ + Starts your stack locally. + """ ctx.run("docker-compose up -d --build") @task(name="stop") -def app_stop(ctx): +def local_stop(ctx): + """ + Stops your local stack. + """ ctx.run("docker-compose down") @task( - name="invoke-function", - help={ - "function_name": "Name of the Lambda to invoke locally (as defined in the Cloudformation template)", - "payload": "JSON payload to include in the trigger event", - }, + name="invoke", + help=_build_help_dict(["function_name", "payload"]), ) -def app_invoke_function(ctx, function_name, payload): +def local_invoke(ctx, function_name, payload): + """ + Triggers the local lambda with the given payload + """ ctx.run( f"aws lambda invoke --endpoint http://localhost:9001 --no-sign-request --function-name {function_name} --log-type Tail --payload {payload} {function_name}_out.json" ) +##################### +# Other invocations # +#################### + + +@task(name="lock") +def lock_requirements(ctx): + """ + Builds the pip lockfile + """ + with ctx.cd(BASE_PATH): + ctx.run("python -m piptools compile requirements.in", hide="both") + ctx.run( + "python -m piptools compile requirements_dev.in --output-file requirements_dev.txt", + hide="both", + ) + + +@task(name="update", help=_build_help_dict(["env", "package"])) +def update_requirements(ctx, env, package): + """ + Updates a package an regenerates the lockfiles. + """ + deps = None + + if env == "prod": + deps = "requirements.in" + elif env == "dev": + deps = "requirements_dev.in" + else: + raise ValueError("Invalid env") + + with ctx.cd(BASE_PATH): + ctx.run(f"python -m piptools compile {deps} --upgrade-package {package}") + + +@task(name="lint", help=_build_help_dict(["fix"])) +def lint(ctx, fix=False): + """ + Lints + """ + with ctx.cd(BASE_PATH): + ctx.run("black *.py **/*.py" + (" --check" if not fix else "")) + + +@task(name="test") +def test(ctx): + """ + Runs tests + """ + with ctx.cd(BASE_PATH): + ctx.run("pytest --cov=src") + + ns = Collection() -# Stack invocations manage the Clouformation flows +local = Collection("local") +local.add_task(local_start) +local.add_task(local_stop) +local.add_task(local_invoke) -stack = Collection("stack") -stack.add_task(stack_deploy) -stack.add_task(teardown_app) -stack.add_task(teardown_bootstrap) +cloud = Collection("cloud") +cloud.add_task(cloud_plan) +cloud.add_task(cloud_apply) +cloud.add_task(cloud_destroy) +cloud.add_task(cloud_pack) +cloud.add_task(cloud_push) -# App invocations manage local containers +project = Collection("requirements") +project.add_task(lock_requirements) +project.add_task(update_requirements) -app = Collection("app") -app.add_task(app_start) -app.add_task(app_stop) -app.add_task(app_invoke_function) +ns.add_collection(local) +ns.add_collection(cloud) +ns.add_collection(project) -ns.add_collection(stack) -ns.add_collection(app) +ns.add_task(lint) +ns.add_task(test)