diff --git a/articles/2019-09-24-automatic-deploy-on-aws.md b/articles/2019-09-24-automatic-deploy-on-aws.md new file mode 100644 index 0000000..88943e9 --- /dev/null +++ b/articles/2019-09-24-automatic-deploy-on-aws.md @@ -0,0 +1,187 @@ +--- +title: Automatic deployment on AWS S3 and/or FTP +date: 2019-09-24 18:30:00 +--- + +A number of websites I maintain are built and deployed automatically through +GitLab pipelines, either on an AWS S3 bucket that is then used by AWS CloudFlare, +or to an SFTP server of an hosting provider. + +In this article I will summarize the necessary configuration to GitLab-CI +and the necessary variables in the CI/CD settings. + +## Deploy to AWS S3 + +We first need to inject the credentials necessary for the GitLab runner to GitLab. +Go to your project, `Settings` -> `CI / CD` -> `Secret variables` and set two variables: + +- `AWS_ACCESS_KEY_ID` with the AWS user's access key +- `AWS_SECRET_ACCESS_KEY` with the AWS user's access secret key + +These variables must not be protected as we want to publish every branch +to a subfolder to be able to preview our branches before merging them. + +The `.gitlab-ci.yml` file will need to contain 3 jobs : Two deployment jobs, +one for branches and one for the master, a third one to cleanup the branch +deployment once the branch is merged or deleted. + +```yaml +image: "python:latest" + +stages: + - deploy + +variables: + S3_BUCKET_NAME: your-bucket-identifier + AWS_BUCKET_REGION: us-east-1 + +deploys3-branch: + stage: deploy + before_script: + - pip install awscli + script: + - aws s3 cp . s3://${S3_BUCKET_NAME}/${CI_COMMIT_REF_SLUG} --recursive --exclude ".gitlab-ci.yml" --exclude ".git/*" + environment: + name: ${CI_COMMIT_REF_SLUG} + url: http://${S3_BUCKET_NAME}.s3-website-${AWS_BUCKET_REGION}.amazonaws.com/${CI_COMMIT_REF_SLUG} + on_stop: deploys3-branch-cleanup + except: + - /^master$/ + +deploys3-branch-cleanup: + stage: deploy + before_script: + - pip install awscli + script: + - aws s3 rm s3://${S3_BUCKET_NAME}/${CI_COMMIT_REF_SLUG} --recursive + environment: + name: ${CI_COMMIT_REF_SLUG} + action: stop + when: manual + +deploys3: + stage: deploy + before_script: + - pip install awscli + script: + - aws s3 cp . s3://${S3_BUCKET_NAME}/ --recursive --exclude ".gitlab-ci.yml" --exclude ".git/*" + environment: + name: ${CI_COMMIT_REF_SLUG} + url: http://${S3_BUCKET_NAME}.s3-website-${AWS_BUCKET_REGION}.amazonaws.com/ + only: + - /^master$/ +``` + +## Deploy to an SFTP server + +This case is somewhat more difficult because of the fact that runner will +need to receive the private key, a multiline string. + +We also need to inject the credentials necessary for the GitLab runner to GitLab. +Go to your project, `Settings` -> `CI / CD` -> `Secret variables` and set two variables: + +- `SFTP_PRIVATE_KEY` with a password-less private key, base64-encoded + +To create the content of this variable, you can simply pipe the content +of your private key into base64 and copy the result into the variable : +`cat project/private-key-deploy_ed25519 | base64 -`. Don't use the same +key that you use for other servers here, always create a new SSH deployment +key! + +These variables must also not be protected as we want to publish every branch +to a subfolder to be able to preview our branches before merging them. + +The `.gitlab-ci.yml` file will also contain 3 jobs : Two deployment jobs, +one for branches and one for the master, a third one to cleanup the branch +deployment once the branch is merged or deleted. + +``` +variables: + SFTP_USER: sftp-user + SFTP_SERVER: sftp.server.domain + SFTP_TARGET: /sftp/server/folder/path + WEBSITE_URL: "https://public.server.domain" + RSYNC_CLI_OPTS: "-a -r" + RSYNC_RSH: 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -a -x' + +## Task Templates +.lftp_template: &lftp_template + image: debian:testing-slim + stage: deploy + before_script: + - apt-get update && apt-get install -q -y rsync openssh-client + - eval $(ssh-agent -s) + - echo ${SFTP_PRIVATE_KEY} | base64 -d - | ssh-add - + +deploysftp-deploy-branches: + <<: *lftp_template + stage: deploy + tags: + - documentation + except: + - master + - /^stable/ + script: + - cd docs + - rsync ${RSYNC_CLI_OPTS} build/ ${SFTP_USER}@${SFTP_SERVER}:${SFTP_TARGET}/${CI_COMMIT_REF_SLUG} + environment: + name: ${CI_COMMIT_REF_SLUG} + url: ${WEBSITE_URL}/${CI_COMMIT_REF_SLUG} + on_stop: deploysftp-clean-doc-branches + when: manual + only: + refs: + - merge_requests + changes: + - docs/**/* + +deploysftp-clean-doc-branches: + <<: *lftp_template + stage: deploy + tags: + - documentation + script: + - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -a -x ${SFTP_USER}@${SFTP_SERVER} "rm -r ${SFTP_TARGET}/${CI_COMMIT_REF_SLUG}" + environment: + name: ${CI_COMMIT_REF_SLUG} + action: stop + when: manual + only: + refs: + - merge_requests + changes: + - docs/**/* + +deploysftp-deploy-master: + <<: *lftp_template + stage: deploy + tags: + - documentation + only: + - master + script: + - cd docs + - rsync ${RSYNC_CLI_OPTS} build/ ${SFTP_USER}@${SFTP_SERVER}:${SFTP_TARGET}/ + environment: + name: ${CI_COMMIT_REF_SLUG} + url: ${WEBSITE_URL}/ +``` + +## Usage + +With these configurations, you can commit your changes in a new branch, +open a merge request and preview your changes deployed as they will be when +the branch is merged. + +The links configured in the two files allow GitLab to present the correct +links in the Merge Request deployment. + +![Deployment](deployment.png "Deployment") + +Once the branch is merged, the deployment is automatically stopped, which +launch the cleanup jobs. + +Using these configuration, you don't have to worry about generating your +website with the latest changes and pushing these changes, the CI/CD process +will automatically do that for you, reducing the number of manual steps +and thus the number of possible errors. diff --git a/articles/2019-09-24-automatic-deploy-on-aws/deployment.png b/articles/2019-09-24-automatic-deploy-on-aws/deployment.png new file mode 100644 index 0000000..e5e1d25 Binary files /dev/null and b/articles/2019-09-24-automatic-deploy-on-aws/deployment.png differ