diff --git a/.github/workflows/docker-ephemeral-env.yml b/.github/workflows/docker-ephemeral-env.yml new file mode 100644 index 0000000000..bfa2542687 --- /dev/null +++ b/.github/workflows/docker-ephemeral-env.yml @@ -0,0 +1,78 @@ +name: Push ephmereral env image + +on: + workflow_run: + workflows: ["Docker"] + types: + - completed + +jobs: + docker_ephemeral_env: + name: Push ephemeral env Docker image to ECR + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + + steps: + - name: 'Download artifact' + uses: actions/github-script@v3.1.0 + with: + script: | + const artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + core.info('*** artifacts') + core.info(JSON.stringify(artifacts)) + + const matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "build" + })[0]; + if(!matchArtifact) return core.setFailed("Build artifacts not found") + + const download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{github.workspace}}/build.zip', Buffer.from(download.data)); + + - run: unzip build.zip + + - name: Display downloaded files (debug) + run: ls -la + + - name: Get SHA + id: get-sha + run: echo "::set-output name=sha::$(cat ./SHA)" + + - name: Get PR + id: get-pr + run: echo "::set-output name=num::$(cat ./PR-NUM)" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Load, tag and push image to ECR + id: push-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: superset-ci + SHA: ${{ steps.get-sha.outputs.sha }} + IMAGE_TAG: pr-${{ steps.get-pr.outputs.num }} + run: | + docker load < $SHA.tar.gz + docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker tag $SHA $ECR_REGISTRY/$ECR_REPOSITORY:$SHA + docker push -a $ECR_REGISTRY/$ECR_REPOSITORY diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7929d812b9..cbbb9a8379 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,3 +24,19 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} run: | .github/workflows/docker_build_push.sh + + - name: Build ephemeral env image + if: github.event_name == 'pull_request' + run: | + mkdir -p ./build + echo ${{ github.sha }} > ./build/SHA + echo ${{ github.event.pull_request.number }} > ./build/PR-NUM + docker build --target ci -t ${{ github.sha }} -t "pr-${{ github.event.pull_request.number }}" . + docker save ${{ github.sha }} | gzip > ./build/${{ github.sha }}.tar.gz + + - name: Upload build artifacts + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v2 + with: + name: build + path: build/ diff --git a/.github/workflows/ecs-task-definition.json b/.github/workflows/ecs-task-definition.json new file mode 100644 index 0000000000..0e4f552898 --- /dev/null +++ b/.github/workflows/ecs-task-definition.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "superset-ci", + "image": "apache/superset:latest", + "cpu": 0, + "links": [], + "portMappings": [ + { + "containerPort": 8080, + "hostPort": 8080, + "protocol": "tcp" + } + ], + "essential": true, + "entryPoint": [], + "command": [], + "environment": [ + { + "name": "SUPERSET_LOAD_EXAMPLES", + "value": "yes" + }, + { + "name": "SUPERSET_PORT", + "value": "8080" + } + ], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/superset-ci", + "awslogs-region": "us-west-2", + "awslogs-stream-prefix": "ecs" + } + } + } + ], + "family": "superset-ci", + "taskRoleArn": "ecsTaskExecutionRole", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [], + "placementConstraints": [], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml new file mode 100644 index 0000000000..ae7a0d6b17 --- /dev/null +++ b/.github/workflows/ephemeral-env.yml @@ -0,0 +1,174 @@ +name: Ephemeral env workflow + +on: + issue_comment: + types: [created] + +jobs: + ephemeral_env_comment: + if: github.event.issue.pull_request + name: Evaluate ephemeral env comment trigger (/testenv) + runs-on: ubuntu-latest + outputs: + slash-command: ${{ steps.eval-body.outputs.result }} + + steps: + - name: Debug + run: | + echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}" + echo "Comment body: ${{ github.event.comment.body }}" + + - name: Eval comment body for /testenv slash command + uses: actions/github-script@v3 + id: eval-body + with: + result-encoding: string + script: | + const pattern = /^\/testenv (up|down)/ + const result = pattern.exec(context.payload.comment.body) + return result === null ? 'noop' : result[1] + + - name: Limit to committers + if: > + steps.eval-body.outputs.result != 'noop' && + github.event.comment.author_association != 'MEMBER' && + github.event.comment.author_association != 'OWNER' + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.' + github.issues.createComment({ + issue_number: ${{ github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: errMsg + }) + core.setFailed(errMsg) + + ephemeral_env_up: + needs: ephemeral_env_comment + if: needs.ephemeral_env_comment.outputs.slash-command == 'up' + name: Spin up an ephemeral environment + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + persist-credentials: false + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Check target image exists in ECR + id: check-image + continue-on-error: true + run: | + aws ecr describe-images \ + --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ + --repository-name superset-ci \ + --image-ids imageTag=pr-${{ github.event.issue.number }} + + - name: Fail on missing container image + if: steps.check-image.outcome == 'failure' + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.' + github.issues.createComment({ + issue_number: ${{ github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: errMsg + }) + core.setFailed(errMsg) + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: .github/workflows/ecs-task-definition.json + container-name: superset-ci + image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }} + + - name: Describe ECS service + id: describe-services + run: | + echo "::set-output name=active::$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" + + - name: Create ECS service + if: steps.describe-services.outputs.active != 'true' + id: create-service + env: + ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974 + ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91 + run: | + aws ecs create-service \ + --cluster superset-ci \ + --service-name pr-${{ github.event.issue.number }}-service \ + --task-definition superset-ci \ + --launch-type FARGATE \ + --desired-count 1 \ + --platform-version LATEST \ + --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \ + --tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }} + + - name: Deploy Amazon ECS task definition + id: deploy-task + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: pr-${{ github.event.issue.number }}-service + cluster: superset-ci + wait-for-service-stability: true + wait-for-minutes: 10 + + - name: List tasks + id: list-tasks + run: | + echo "::set-output name=task::$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" + + - name: Get network interface + id: get-eni + run: | + echo "::set-output name=eni::$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" + + - name: Get public IP + id: get-ip + run: | + echo "::set-output name=ip::$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" + + - name: Comment (success) + if: ${{ success() }} + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.issues.createComment({ + issue_number: ${{ github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.' + }) + + - name: Comment (failure) + if: ${{ failure() }} + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + github.issues.createComment({ + issue_number: ${{ github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.' + }) diff --git a/Dockerfile b/Dockerfile index bf64199923..bf120d756c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -131,3 +131,17 @@ RUN cd /app \ && pip install --no-cache -r requirements/docker.txt \ && pip install --no-cache -r requirements/requirements-local.txt || true USER superset + + +###################################################################### +# CI image... +###################################################################### +FROM lean AS ci + +COPY --chown=superset ./docker/docker-bootstrap.sh /app/docker/ +COPY --chown=superset ./docker/docker-init.sh /app/docker/ +COPY --chown=superset ./docker/docker-ci.sh /app/docker/ + +RUN chmod a+x /app/docker/*.sh + +CMD /app/docker/docker-ci.sh diff --git a/docker/docker-ci.sh b/docker/docker-ci.sh new file mode 100755 index 0000000000..cc84c8adc9 --- /dev/null +++ b/docker/docker-ci.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +/app/docker/docker-init.sh + +# TODO: copy config overrides from ENV vars + +# TODO: run celery in detached state + +# start up the web server +gunicorn \ + --bind "0.0.0.0:${SUPERSET_PORT}" \ + --access-logfile '-' \ + --error-logfile '-' \ + --workers 1 \ + --worker-class gthread \ + --threads 8 \ + --timeout 60 \ + --limit-request-line 0 \ + --limit-request-field_size 0 \ + "${FLASK_APP}"