From d2a6cfe37818ed69f683467a3f1b6fe9505a1fb2 Mon Sep 17 00:00:00 2001 From: Marc Cataford Date: Sat, 26 Sep 2020 01:14:56 -0400 Subject: [PATCH] Initial version (#1) * infra: python, bootstrap * feat: local sample * infra: invocations and CFN * docs: stub * wip: clean up, add sceptre * wip: add app stack teardown * refactor: template rejigging * chore: ignore outfiles * wip: cleanup post upload to s3 * wip: teardown infra task * docs: links, contrib * docs: missing command * docs: formatting --- .gitignore | 4 + .python-version | 1 + Dockerfile | 7 ++ README.md | 39 ++++++++ dev_requirements.txt | 30 ++++++ docker-compose.yml | 13 +++ infrastructure/config/app/app.yaml | 8 ++ infrastructure/config/app/config.yaml | 2 + .../config/bootstrap/bootstrap.yaml | 4 + infrastructure/config/bootstrap/config.yaml | 2 + infrastructure/config/config.yaml | 2 + infrastructure/templates/app.yaml | 99 +++++++++++++++++++ infrastructure/templates/bootstrap.yaml | 18 ++++ script/bootstrap | 15 +++ src/__init__.py | 0 src/base.py | 5 + tasks.py | 87 ++++++++++++++++ 17 files changed, 336 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 Dockerfile create mode 100644 dev_requirements.txt create mode 100644 docker-compose.yml create mode 100644 infrastructure/config/app/app.yaml create mode 100644 infrastructure/config/app/config.yaml create mode 100644 infrastructure/config/bootstrap/bootstrap.yaml create mode 100644 infrastructure/config/bootstrap/config.yaml create mode 100644 infrastructure/config/config.yaml create mode 100644 infrastructure/templates/app.yaml create mode 100644 infrastructure/templates/bootstrap.yaml create mode 100644 script/bootstrap create mode 100644 src/__init__.py create mode 100644 src/base.py create mode 100644 tasks.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0de1eff --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.sw* +*.pyc +*.zip +*_out.json diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..a08ffae --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.8.2 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9d50165 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM lambci/lambda:python3.8 + +USER root + +ENV APP_DIR /var/task + +WORKDIR $APP_DIR diff --git a/README.md b/README.md index ee2b749..ddda1e5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,41 @@ # lambda-boilerplate 🛠 Skip the boilerplate and start building fun λ things 🛠 + +## Overview + +AWS Lambdas are fun, but often the amount of boilerplate involved in getting a project off the ground hinders the fun. From setting up a local environment to writing out a Cloudformation template, the overhead of Lambda-based greenfield projects can be daunting. No more. Just use this repository as a template or clone it and jump straight into the action! This repository offers a quick template your can use, full with a Docker setup for local development and invocation commands that you can use to package and deploy small Lambdas. + +Use this as a foundation and tweak it to your use case! + +## Local development + +To get started, hit the bootstrap script with `. script/bootstrap`. This will set up a Python 3.8 virtualenv set up with some basic tools that will make your life easier. + +The base Lambda handler is at `src/base.py` and all the infrastructure templates and Sceptre configuration are in `infrastructure`. + +[Read more about Sceptre](https://sceptre.cloudreach.com/latest/index.html) + +[Read more about AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html) + +### Invocations + +This template uses PyInvoke, all commands are of the format `inv `. + +|Command|Description| +|---|---| +|`app.start`|Start your Lambda in Docker.| +|`app.stop`|Stop and remove the container.| +|`app.invoke-function `|Invokes the given local Lambda by container name| +|`stack.deploy`|Packages your code and deploys the stack| +|`stack.teardown-app`|Tears down the application stack| +|`stack.teardown-bootstrap`|Tears down the bootstrap stack| + +## Deployment + +The base setup assumes that your Lambda handler is located in `src.base`. Doing `inv stack.deploy` will zip up your `src` directory, create an S3 bucket for your development artifacts, uploads the source doe archive to S3 and kickstarts the Cloudformation deployment of your stack. + +## Contributing + +Got suggestions or improvements you'd like to make? Open a PR or [an issue](https://github.com/mcataford/lambda-boilerplate/issues)! + +Feature requests should keep in mind that the goal of this boilerplate is to be general enough so that the cost of tailoring it to specific use cases is low. diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..ee86a1a --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,30 @@ +appdirs==1.4.4 +attrs==20.1.0 +awscli==1.18.124 +black==19.10b0 +boto3==1.14.55 +botocore==1.17.55 +click==7.1.2 +colorama==0.3.9 +decorator==4.4.2 +docutils==0.15.2 +invoke==1.4.1 +Jinja2==2.11.2 +jmespath==0.10.0 +MarkupSafe==1.1.1 +networkx==2.1 +packaging==16.8 +pathspec==0.8.0 +pyasn1==0.4.8 +pyparsing==2.4.7 +python-dateutil==2.8.1 +PyYAML==5.3.1 +regex==2020.7.14 +rsa==4.5 +s3transfer==0.3.3 +sceptre==2.3.0 +six==1.15.0 +toml==0.10.1 +typed-ast==1.4.1 +typing==3.7.4.3 +urllib3==1.25.10 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..09b0028 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.2' +services: + web: + build: . + volumes: + - ./src/:/var/task/ + command: "base.handler" + ports: + - "9001:9001" + environment: + PYTHONPATH: /var/task/src:/var/task/lib + DOCKER_LAMBDA_STAY_OPEN: 1 + DOCKER_LAMBDA_WATCH: 1 diff --git a/infrastructure/config/app/app.yaml b/infrastructure/config/app/app.yaml new file mode 100644 index 0000000..3e68714 --- /dev/null +++ b/infrastructure/config/app/app.yaml @@ -0,0 +1,8 @@ +template_path: app.yaml +parameters: + # Lambda source artifacts bucket name + ArtifactsBucketName: !stack_output bootstrap/bootstrap.yaml::ArtifactsBucketNameTest + # Lambda zip key + SourceKey: {{ var.source_key | default("") }} + ApiStageName: "v0" + diff --git a/infrastructure/config/app/config.yaml b/infrastructure/config/app/config.yaml new file mode 100644 index 0000000..6174c26 --- /dev/null +++ b/infrastructure/config/app/config.yaml @@ -0,0 +1,2 @@ +project_code: app +region: us-east-1 diff --git a/infrastructure/config/bootstrap/bootstrap.yaml b/infrastructure/config/bootstrap/bootstrap.yaml new file mode 100644 index 0000000..38b54c1 --- /dev/null +++ b/infrastructure/config/bootstrap/bootstrap.yaml @@ -0,0 +1,4 @@ +template_path: bootstrap.yaml +parameters: + # Bucket name for the artifacts S# bucket. Used solely to store versioned lambda zips. + ArtifactsBucketName: my-cool-bucket-name diff --git a/infrastructure/config/bootstrap/config.yaml b/infrastructure/config/bootstrap/config.yaml new file mode 100644 index 0000000..6174c26 --- /dev/null +++ b/infrastructure/config/bootstrap/config.yaml @@ -0,0 +1,2 @@ +project_code: app +region: us-east-1 diff --git a/infrastructure/config/config.yaml b/infrastructure/config/config.yaml new file mode 100644 index 0000000..6174c26 --- /dev/null +++ b/infrastructure/config/config.yaml @@ -0,0 +1,2 @@ +project_code: app +region: us-east-1 diff --git a/infrastructure/templates/app.yaml b/infrastructure/templates/app.yaml new file mode 100644 index 0000000..40c0cf9 --- /dev/null +++ b/infrastructure/templates/app.yaml @@ -0,0 +1,99 @@ +AWSTemplateFormatVersion: 2010-09-09 +Parameters: + ArtifactsBucketName: + Type: String + Description: Bucket storing the function source. + SourceKey: + Type: String + ApiStageName: + Type: String +Resources: + # Lambda function + Function: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: !Ref ArtifactsBucketName + S3Key: !Ref SourceKey + FunctionName: sampleFunction + Handler: src.base.handler + Role: !GetAtt LambdaExecutionRole.Arn + Runtime: python3.8 + # Roles + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + ApiGatewayRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - apigateway.amazonaws.com + Action: + - sts:AssumeRole + Path: '/' + Policies: + - PolicyName: LambdaAccess + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: 'lambda:*' + Resource: !GetAtt Function.Arn + + + RestApi: + Type: AWS::ApiGateway::RestApi + Properties: + Name: lambda-api + GatewayResource: + Type: AWS::ApiGateway::Resource + Properties: + ParentId: !GetAtt RestApi.RootResourceId + PathPart: lambda + RestApiId: !Ref RestApi + # This template only defines a POST method, but others can easily be defined + # by duping this resource and tweaking it for other resources, opnames or URIs. + PostMethod: + Type: AWS::ApiGateway::Method + Properties: + HttpMethod: POST + AuthorizationType: NONE + Integration: + Credentials: !GetAtt ApiGatewayRole.Arn + IntegrationHttpMethod: POST + Type: AWS_PROXY + Uri: !Sub 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${Function.Arn}/invocations' + OperationName: lambda + ResourceId: !Ref GatewayResource + RestApiId: !Ref RestApi + ApiStage: + Type: AWS::ApiGateway::Stage + Properties: + DeploymentId: !Ref ApiDeployment + RestApiId: !Ref RestApi + StageName: !Ref ApiStageName + ApiDeployment: + Type: AWS::ApiGateway::Deployment + DependsOn: PostMethod + Properties: + RestApiId: !Ref RestApi + +Outputs: + ApiGatewayId: + Value: !Ref RestApi + Export: + Name: RestApiId diff --git a/infrastructure/templates/bootstrap.yaml b/infrastructure/templates/bootstrap.yaml new file mode 100644 index 0000000..e9811a8 --- /dev/null +++ b/infrastructure/templates/bootstrap.yaml @@ -0,0 +1,18 @@ +AWSTemplateFormatVersion: 2010-09-09 +Parameters: + ArtifactsBucketName: + Type: String + Description: Bucket storing the function source +Resources: + DevBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref ArtifactsBucketName + AccessControl: Private + VersioningConfiguration: + Status: Enabled +Outputs: + ArtifactsBucketNameTest: + Value: !Ref ArtifactsBucketName + Export: + Name: ArtifactsBucketNameTest diff --git a/script/bootstrap b/script/bootstrap new file mode 100644 index 0000000..01db817 --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,15 @@ +VENV=lambda-boilerplate.venv + +################################################################# +# Bootstrapping sets up the Python 3.8 venv that allows the use # +# of the invoke commands. # +################################################################# + +{ + pyenv virtualenv-delete -f $VENV + pyenv virtualenv $VENV && + pyenv activate $VENV && + python -m pip install -U pip && + pip install -r dev_requirements.txt && + echo "✨ Good to go! ✨" +} diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/base.py b/src/base.py new file mode 100644 index 0000000..71408b2 --- /dev/null +++ b/src/base.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + + +def handler(event, context): + return {"statusCode": 200} diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..8941ef3 --- /dev/null +++ b/tasks.py @@ -0,0 +1,87 @@ +from invoke import task, Collection +import boto3 +import os +from pathlib import Path +import hashlib + +##################### +# Stack invocations # +##################### + + +@task(name="teardown-app") +def teardown_app(ctx): + with ctx.cd("infrastructure"): + ctx.run("sceptre delete app/app.yaml -y") + +@task(name="teardown-bootstrap") +def teardown_bootstrap(ctx): + with ctx.cd("infrastructure"): + ctx.run("sceptre delete bootstrap/bootstrap.yaml -y") + + + +@task(name="deploy") +def stack_deploy(ctx): + path = Path(__file__).parent + with ctx.cd("infrastructure"): + ctx.run("sceptre launch bootstrap/bootstrap.yaml -y") + + ctx.run("zip lambda_function.zip src/*") + with open("lambda_function.zip", "rb") as src: + srchash = hashlib.md5(src.read()).hexdigest() + new_archive_name = f"lambda_function_{srchash}.zip" + ctx.run(f"mv lambda_function.zip {new_archive_name}") + + ctx.run(f"aws s3 cp {new_archive_name} s3://mcat-dev-test-bucket-artifacts-2 && rm {new_archive_name}") + + with ctx.cd("infrastructure"): + ctx.run(f"sceptre --var source_key={new_archive_name} launch app/app.yaml -y") + + +##################### +# Local invocations # +##################### + + +@task(name="start") +def app_start(ctx): + ctx.run("docker-compose up -d --build") + + +@task(name="stop") +def app_stop(ctx): + ctx.run("docker-compose down") + + +@task( + name="invoke-function", + help={ + "function_name": "Name of the Lambda to invoke locally (as defined in the Cloudformation template)", + "payload": "JSON payload to include in the trigger event", + }, +) +def app_invoke_function(ctx, function_name, payload): + ctx.run( + f"aws lambda invoke --endpoint http://localhost:9001 --no-sign-request --function-name {function_name} --log-type Tail --payload {payload} {function_name}_out.json" + ) + + +ns = Collection() + +# Stack invocations manage the Clouformation flows + +stack = Collection("stack") +stack.add_task(stack_deploy) +stack.add_task(teardown_app) +stack.add_task(teardown_bootstrap) + +# App invocations manage local containers + +app = Collection("app") +app.add_task(app_start) +app.add_task(app_stop) +app.add_task(app_invoke_function) + +ns.add_collection(stack) +ns.add_collection(app)