2020-04-14 16:38:09 -04:00
|
|
|
#!/bin/bash
|
|
|
|
#
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
|
|
# this work for additional information regarding copyright ownership.
|
|
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
|
|
# (the "License"); you may not use this file except in compliance with
|
|
|
|
# the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
#
|
2020-04-17 02:35:01 -04:00
|
|
|
set -e
|
|
|
|
|
|
|
|
GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
|
|
|
|
ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
|
2020-04-14 16:38:09 -04:00
|
|
|
|
2020-07-31 01:28:56 -04:00
|
|
|
# Rounded job start time, used to create a unique Cypress build id for
|
|
|
|
# parallelization so we can manually rerun a job after 20 minutes
|
|
|
|
NONCE=$(echo "$(date "+%Y%m%d%H%M") - ($(date +%M)%20)" | bc)
|
|
|
|
|
2020-04-14 16:38:09 -04:00
|
|
|
# Echo only when not in parallel mode
|
|
|
|
say() {
|
2020-04-17 02:35:01 -04:00
|
|
|
if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
|
2020-04-14 16:38:09 -04:00
|
|
|
echo "$1"
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
# default command to run when the `run` input is empty
|
|
|
|
default-setup-command() {
|
2020-08-04 22:31:52 -04:00
|
|
|
apt-get-install
|
|
|
|
pip-upgrade
|
2020-04-14 16:38:09 -04:00
|
|
|
}
|
|
|
|
|
2020-08-04 22:31:52 -04:00
|
|
|
apt-get-install() {
|
|
|
|
say "::group::apt-get install dependencies"
|
|
|
|
sudo apt-get update && sudo apt-get install --yes \
|
|
|
|
libsasl2-dev
|
|
|
|
say "::endgroup::"
|
|
|
|
}
|
2020-04-14 16:38:09 -04:00
|
|
|
|
2020-08-04 22:31:52 -04:00
|
|
|
pip-upgrade() {
|
|
|
|
say "::group::Upgrade pip"
|
2020-07-31 01:28:56 -04:00
|
|
|
pip install --upgrade pip
|
2020-04-14 16:38:09 -04:00
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
|
|
|
# prepare (lint and build) frontend code
|
|
|
|
npm-install() {
|
|
|
|
cd "$GITHUB_WORKSPACE/superset-frontend"
|
|
|
|
|
2020-07-31 01:28:56 -04:00
|
|
|
# cache-restore npm
|
2020-04-14 16:38:09 -04:00
|
|
|
say "::group::Install npm packages"
|
|
|
|
echo "npm: $(npm --version)"
|
|
|
|
echo "node: $(node --version)"
|
2020-08-29 21:09:47 -04:00
|
|
|
npm ci
|
2020-04-14 16:38:09 -04:00
|
|
|
say "::endgroup::"
|
|
|
|
|
2020-07-31 01:28:56 -04:00
|
|
|
# cache-save npm
|
2020-04-14 16:38:09 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
build-assets() {
|
|
|
|
cd "$GITHUB_WORKSPACE/superset-frontend"
|
|
|
|
|
|
|
|
say "::group::Build static assets"
|
2021-09-22 07:24:54 -04:00
|
|
|
npm run build
|
2020-04-14 16:38:09 -04:00
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
2020-04-17 02:35:01 -04:00
|
|
|
build-instrumented-assets() {
|
|
|
|
cd "$GITHUB_WORKSPACE/superset-frontend"
|
2020-04-14 16:38:09 -04:00
|
|
|
|
2020-04-17 02:35:01 -04:00
|
|
|
say "::group::Build static assets with JS instrumented for test coverage"
|
|
|
|
cache-restore instrumented-assets
|
|
|
|
if [[ -f "$ASSETS_MANIFEST" ]]; then
|
|
|
|
echo 'Skip frontend build because instrumented static assets already exist.'
|
|
|
|
else
|
2021-09-22 07:24:54 -04:00
|
|
|
npm run build-instrumented
|
2020-04-17 02:35:01 -04:00
|
|
|
cache-save instrumented-assets
|
|
|
|
fi
|
2020-04-14 16:38:09 -04:00
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
|
|
|
setup-postgres() {
|
2020-12-04 22:17:23 -05:00
|
|
|
say "::group::Install dependency for unit tests"
|
|
|
|
sudo apt-get update && sudo apt-get install --yes libecpg-dev
|
2020-04-14 16:38:09 -04:00
|
|
|
say "::group::Initialize database"
|
|
|
|
psql "postgresql://superset:superset@127.0.0.1:15432/superset" <<-EOF
|
2020-10-13 13:31:49 -04:00
|
|
|
DROP SCHEMA IF EXISTS sqllab_test_db CASCADE;
|
|
|
|
DROP SCHEMA IF EXISTS admin_database CASCADE;
|
2020-04-14 16:38:09 -04:00
|
|
|
CREATE SCHEMA sqllab_test_db;
|
|
|
|
CREATE SCHEMA admin_database;
|
|
|
|
EOF
|
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
|
|
|
setup-mysql() {
|
|
|
|
say "::group::Initialize database"
|
|
|
|
mysql -h 127.0.0.1 -P 13306 -u root --password=root <<-EOF
|
|
|
|
DROP DATABASE IF EXISTS superset;
|
|
|
|
CREATE DATABASE superset DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
|
|
|
DROP DATABASE IF EXISTS sqllab_test_db;
|
|
|
|
CREATE DATABASE sqllab_test_db DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
|
|
|
DROP DATABASE IF EXISTS admin_database;
|
|
|
|
CREATE DATABASE admin_database DEFAULT CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
|
|
|
CREATE USER 'superset'@'%' IDENTIFIED BY 'superset';
|
|
|
|
GRANT ALL ON *.* TO 'superset'@'%';
|
|
|
|
FLUSH PRIVILEGES;
|
|
|
|
EOF
|
|
|
|
say "::endgroup::"
|
|
|
|
}
|
2020-04-17 02:35:01 -04:00
|
|
|
|
|
|
|
testdata() {
|
|
|
|
cd "$GITHUB_WORKSPACE"
|
|
|
|
say "::group::Load test data"
|
|
|
|
# must specify PYTHONPATH to make `tests.superset_test_config` importable
|
|
|
|
export PYTHONPATH="$GITHUB_WORKSPACE"
|
2020-08-04 22:31:52 -04:00
|
|
|
pip install -e .
|
2020-04-17 02:35:01 -04:00
|
|
|
superset db upgrade
|
|
|
|
superset load_test_users
|
|
|
|
superset load_examples --load-test-data
|
|
|
|
superset init
|
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
|
|
|
codecov() {
|
|
|
|
say "::group::Upload code coverage"
|
2021-04-16 12:26:50 -04:00
|
|
|
bash ".github/workflows/codecov.sh" "$@"
|
2020-04-17 02:35:01 -04:00
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
|
|
|
cypress-install() {
|
|
|
|
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
|
|
|
|
|
|
|
|
cache-restore cypress
|
|
|
|
|
|
|
|
say "::group::Install Cypress"
|
|
|
|
npm ci
|
|
|
|
say "::endgroup::"
|
|
|
|
|
|
|
|
cache-save cypress
|
|
|
|
}
|
|
|
|
|
|
|
|
# Run Cypress and upload coverage reports
|
|
|
|
cypress-run() {
|
|
|
|
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
|
2020-07-31 01:28:56 -04:00
|
|
|
|
2020-04-17 02:35:01 -04:00
|
|
|
local page=$1
|
|
|
|
local group=${2:-Default}
|
|
|
|
local cypress="./node_modules/.bin/cypress run"
|
|
|
|
local browser=${CYPRESS_BROWSER:-chrome}
|
|
|
|
|
2020-07-31 01:28:56 -04:00
|
|
|
export TERM="xterm"
|
|
|
|
|
2020-04-17 02:35:01 -04:00
|
|
|
say "::group::Run Cypress for [$page]"
|
2021-01-15 15:18:18 -05:00
|
|
|
if [[ -z $CYPRESS_KEY ]]; then
|
2020-04-17 02:35:01 -04:00
|
|
|
$cypress --spec "cypress/integration/$page" --browser "$browser"
|
|
|
|
else
|
2021-01-15 15:18:18 -05:00
|
|
|
export CYPRESS_RECORD_KEY=`echo $CYPRESS_KEY | base64 --decode`
|
2020-04-17 02:35:01 -04:00
|
|
|
# additional flags for Cypress dashboard recording
|
2020-07-31 01:28:56 -04:00
|
|
|
$cypress --spec "cypress/integration/$page" --browser "$browser" \
|
|
|
|
--record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
|
|
|
|
--parallel --ci-build-id "${GITHUB_SHA:0:8}-${NONCE}"
|
2020-04-17 02:35:01 -04:00
|
|
|
fi
|
|
|
|
|
|
|
|
# don't add quotes to $record because we do want word splitting
|
|
|
|
say "::endgroup::"
|
|
|
|
}
|
|
|
|
|
|
|
|
cypress-run-all() {
|
|
|
|
# Start Flask and run it in background
|
|
|
|
# --no-debugger means disable the interactive debugger on the 500 page
|
|
|
|
# so errors can print to stderr.
|
|
|
|
local flasklog="${HOME}/flask.log"
|
|
|
|
local port=8081
|
2020-09-16 15:31:03 -04:00
|
|
|
export CYPRESS_BASE_URL="http://localhost:${port}"
|
2020-04-17 02:35:01 -04:00
|
|
|
|
2020-07-31 01:28:56 -04:00
|
|
|
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
|
2020-04-17 02:35:01 -04:00
|
|
|
local flaskProcessId=$!
|
|
|
|
|
2020-06-30 20:10:17 -04:00
|
|
|
cypress-run "*/**/*"
|
2020-04-17 02:35:01 -04:00
|
|
|
|
|
|
|
# After job is done, print out Flask log for debugging
|
|
|
|
say "::group::Flask log for default run"
|
|
|
|
cat "$flasklog"
|
|
|
|
say "::endgroup::"
|
|
|
|
|
|
|
|
# Rerun SQL Lab tests with backend persist enabled
|
2021-07-01 11:03:07 -04:00
|
|
|
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config_sqllab_backend_persist
|
2020-04-17 02:35:01 -04:00
|
|
|
|
|
|
|
# Restart Flask with new configs
|
|
|
|
kill $flaskProcessId
|
2020-07-31 01:28:56 -04:00
|
|
|
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
|
2020-04-17 02:35:01 -04:00
|
|
|
local flaskProcessId=$!
|
|
|
|
|
|
|
|
cypress-run "sqllab/*" "Backend persist"
|
2020-09-15 17:31:43 -04:00
|
|
|
|
2021-02-25 08:22:34 -05:00
|
|
|
# Upload code coverage separately so each page can have separate flags
|
|
|
|
# -c will clean existing coverage reports, -F means add flags
|
2020-09-15 17:31:43 -04:00
|
|
|
# || true to prevent CI failure on codecov upload
|
2021-02-25 08:22:34 -05:00
|
|
|
codecov -c -F "cypress" || true
|
2020-04-17 02:35:01 -04:00
|
|
|
|
|
|
|
say "::group::Flask log for backend persist"
|
|
|
|
cat "$flasklog"
|
|
|
|
say "::endgroup::"
|
|
|
|
|
|
|
|
# make sure the program exits
|
|
|
|
kill $flaskProcessId
|
|
|
|
}
|