pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci: conditionally skip python and frontend tests (#12583) 

* ci: conditionally skip python tests

* parametrize check script and implement for frontend

* add missing script

* fix incorrect check

* move CHECKS to ARGS and fix typo

* fix SC2086 nit
This commit is contained in:
Ville Brofeldt 2021-02-05 17:40:08 +02:00 committed by GitHub
parent 3235d918aa
commit 5d55c4367c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 137 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/superset-frontend.yml vendored
View File

@ -18,20 +18,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if frontend changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh frontend
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run lint
npm run prettier-check
- name: unit tests
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run test -- --coverage
- name: Upload code coverage
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
bash <(curl -s https://codecov.io/bash) -cF javascript

9
.github/workflows/superset-python-misc.yml vendored
View File

@ -21,11 +21,19 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
@ -33,6 +41,7 @@ jobs:
pip-upgrade
pip install -r requirements/testing.txt
- name: pylint
if: steps.check.outcome == 'failure'
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset

27
.github/workflows/superset-python-presto-hive.yml vendored
View File

@ -51,11 +51,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
@ -64,11 +73,14 @@ jobs:
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
@ -106,17 +118,29 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Create csv upload directory
if: steps.check.outcome == 'failure'
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
if: steps.check.outcome == 'failure'
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
if: steps.check.outcome == 'failure'
run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
@ -125,10 +149,13 @@ jobs:
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python

36
.github/workflows/superset-python-unittest.yml vendored
View File

@ -39,11 +39,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
@ -52,11 +61,14 @@ jobs:
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
@ -91,11 +103,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
@ -104,11 +125,14 @@ jobs:
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
@ -135,11 +159,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
@ -148,10 +181,13 @@ jobs:
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python

54
scripts/ci_check_no_file_changes.sh Executable file
View File

@ -0,0 +1,54 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Usage instructions:
#
# to check for python changes, run with CHECKS=python
# To check for frontend changes, run with CHECKS=frontend
# To check for python and frontend changes, run with CHECKS="python frontend"
URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files"
FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename')
cat<<EOF
CHANGED FILES:
$FILES
EOF
for CHECK in "$@"
do
if [[ ${CHECK} == "python" ]]; then
REGEX="(^tests\/|^superset\/|^setup\.py|^requirements\/.+\.txt)"
echo "Searching for changes in python files"
elif [[ ${CHECK} == "frontend" ]]; then
REGEX="(^superset-frontend\/)"
echo "Searching for changes in frontend files"
else
echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code"
exit 1
fi
if [[ "${FILES}" =~ ${REGEX} ]]; then
echo "Detected changes... Exiting with FAILURE code"
exit 1
fi
done
echo "No changes detected... Exiting with SUCCESS code"
exit 0