2020-11-24 14:40:16 -05:00
|
|
|
# Python Presto/Hive unit tests
|
|
|
|
name: Python Presto/Hive
|
|
|
|
|
2021-01-07 03:47:00 -05:00
|
|
|
on:
|
|
|
|
push:
|
|
|
|
branches-ignore:
|
|
|
|
- "dependabot/npm_and_yarn/**"
|
|
|
|
pull_request:
|
2021-01-21 19:10:17 -05:00
|
|
|
types: [synchronize, opened, reopened, ready_for_review]
|
2020-11-24 14:40:16 -05:00
|
|
|
|
|
|
|
jobs:
|
|
|
|
test-postgres-presto:
|
2021-01-21 19:10:17 -05:00
|
|
|
if: github.event.pull_request.draft == false
|
2021-01-15 19:14:33 -05:00
|
|
|
runs-on: ubuntu-20.04
|
2020-11-24 14:40:16 -05:00
|
|
|
strategy:
|
|
|
|
matrix:
|
|
|
|
python-version: [3.8]
|
|
|
|
env:
|
|
|
|
PYTHONPATH: ${{ github.workspace }}
|
2021-07-01 11:03:07 -04:00
|
|
|
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
2020-11-24 14:40:16 -05:00
|
|
|
REDIS_PORT: 16379
|
|
|
|
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
|
|
|
SUPERSET__SQLALCHEMY_EXAMPLES_URI: presto://localhost:15433/memory/default
|
|
|
|
services:
|
|
|
|
postgres:
|
|
|
|
image: postgres:10-alpine
|
|
|
|
env:
|
|
|
|
POSTGRES_USER: superset
|
|
|
|
POSTGRES_PASSWORD: superset
|
|
|
|
ports:
|
|
|
|
# Use custom ports for services to avoid accidentally connecting to
|
|
|
|
# GitHub action runner's default installations
|
|
|
|
- 15432:5432
|
|
|
|
presto:
|
2021-03-23 23:11:58 -04:00
|
|
|
image: starburstdata/presto:350-e.6
|
2020-11-24 14:40:16 -05:00
|
|
|
env:
|
|
|
|
POSTGRES_USER: superset
|
|
|
|
POSTGRES_PASSWORD: superset
|
|
|
|
ports:
|
|
|
|
# Use custom ports for services to avoid accidentally connecting to
|
|
|
|
# GitHub action runner's default installations
|
|
|
|
- 15433:8080
|
|
|
|
redis:
|
|
|
|
image: redis:5-alpine
|
|
|
|
ports:
|
|
|
|
- 16379:6379
|
|
|
|
steps:
|
2021-01-24 02:10:16 -05:00
|
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
|
|
uses: actions/checkout@v2
|
|
|
|
with:
|
|
|
|
persist-credentials: false
|
|
|
|
submodules: recursive
|
2021-02-05 10:40:08 -05:00
|
|
|
- name: Check if python changes are present
|
|
|
|
id: check
|
|
|
|
env:
|
|
|
|
GITHUB_REPO: ${{ github.repository }}
|
|
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
|
|
continue-on-error: true
|
|
|
|
run: ./scripts/ci_check_no_file_changes.sh python
|
2020-11-24 14:40:16 -05:00
|
|
|
- name: Setup Python
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
uses: actions/setup-python@v2
|
|
|
|
with:
|
|
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2021-01-04 07:16:07 -05:00
|
|
|
uses: ./.github/actions/cached-dependencies
|
2020-11-24 14:40:16 -05:00
|
|
|
with:
|
|
|
|
run: |
|
|
|
|
apt-get-install
|
|
|
|
pip-upgrade
|
|
|
|
pip install -r requirements/testing.txt
|
|
|
|
setup-postgres
|
|
|
|
- name: Run celery
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2021-06-08 23:18:55 -04:00
|
|
|
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
2020-11-24 14:40:16 -05:00
|
|
|
- name: Python unit tests (PostgreSQL)
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: |
|
|
|
|
./scripts/python_tests.sh
|
|
|
|
- name: Upload code coverage
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: |
|
2021-04-16 12:26:50 -04:00
|
|
|
bash .github/workflows/codecov.sh -c -F python -F presto
|
2020-11-24 14:40:16 -05:00
|
|
|
|
|
|
|
test-postgres-hive:
|
2021-01-21 19:10:17 -05:00
|
|
|
if: github.event.pull_request.draft == false
|
2021-01-15 19:14:33 -05:00
|
|
|
runs-on: ubuntu-20.04
|
2020-11-24 14:40:16 -05:00
|
|
|
strategy:
|
|
|
|
matrix:
|
2021-03-26 02:03:58 -04:00
|
|
|
python-version: [3.8]
|
2020-11-24 14:40:16 -05:00
|
|
|
env:
|
|
|
|
PYTHONPATH: ${{ github.workspace }}
|
2021-07-01 11:03:07 -04:00
|
|
|
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
2020-11-24 14:40:16 -05:00
|
|
|
REDIS_PORT: 16379
|
|
|
|
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
|
|
|
SUPERSET__SQLALCHEMY_EXAMPLES_URI: hive://localhost:10000/default
|
|
|
|
UPLOAD_FOLDER: /tmp/.superset/uploads/
|
|
|
|
services:
|
|
|
|
postgres:
|
|
|
|
image: postgres:10-alpine
|
|
|
|
env:
|
|
|
|
POSTGRES_USER: superset
|
|
|
|
POSTGRES_PASSWORD: superset
|
|
|
|
ports:
|
|
|
|
# Use custom ports for services to avoid accidentally connecting to
|
|
|
|
# GitHub action runner's default installations
|
|
|
|
- 15432:5432
|
|
|
|
redis:
|
|
|
|
image: redis:5-alpine
|
|
|
|
ports:
|
|
|
|
- 16379:6379
|
|
|
|
steps:
|
2021-01-24 02:10:16 -05:00
|
|
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
|
|
uses: actions/checkout@v2
|
|
|
|
with:
|
|
|
|
persist-credentials: false
|
|
|
|
submodules: recursive
|
2021-02-05 10:40:08 -05:00
|
|
|
- name: Check if python changes are present
|
|
|
|
id: check
|
|
|
|
env:
|
|
|
|
GITHUB_REPO: ${{ github.repository }}
|
|
|
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
|
|
continue-on-error: true
|
|
|
|
run: ./scripts/ci_check_no_file_changes.sh python
|
2020-11-24 14:40:16 -05:00
|
|
|
- name: Create csv upload directory
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: sudo mkdir -p /tmp/.superset/uploads
|
|
|
|
- name: Give write access to the csv upload directory
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: sudo chown -R $USER:$USER /tmp/.superset
|
|
|
|
- name: Start hadoop and hive
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
|
|
|
|
- name: Setup Python
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
uses: actions/setup-python@v2
|
|
|
|
with:
|
|
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install dependencies
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2021-01-04 07:16:07 -05:00
|
|
|
uses: ./.github/actions/cached-dependencies
|
2020-11-24 14:40:16 -05:00
|
|
|
with:
|
|
|
|
run: |
|
|
|
|
apt-get-install
|
|
|
|
pip-upgrade
|
|
|
|
pip install -r requirements/testing.txt
|
|
|
|
setup-postgres
|
|
|
|
- name: Run celery
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2021-06-08 23:18:55 -04:00
|
|
|
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
2020-11-24 14:40:16 -05:00
|
|
|
- name: Python unit tests (PostgreSQL)
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: |
|
|
|
|
./scripts/python_tests.sh
|
|
|
|
- name: Upload code coverage
|
2021-02-05 10:40:08 -05:00
|
|
|
if: steps.check.outcome == 'failure'
|
2020-11-24 14:40:16 -05:00
|
|
|
run: |
|
2021-04-16 12:26:50 -04:00
|
|
|
bash .github/workflows/codecov.sh -c -F python -F hive
|