pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

tests(celery): improve celery tests infra (#9775)

This commit is contained in:
Daniel Vaz Gaspar 2020-05-14 16:58:30 +01:00 committed by GitHub
parent d02f2d1fa7
commit 4427d65717
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 100 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.github/workflows/superset-python.yml vendored
View File

@ -61,6 +61,8 @@ jobs:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI:
postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
services:
postgres:
image: postgres:10-alpine
@ -87,10 +89,9 @@ jobs:
run: |
pip-install
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
env:
SUPERSET__SQLALCHEMY_DATABASE_URI:
postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
run: |
./scripts/python_tests.sh
- name: Upload code coverage
@ -106,6 +107,8 @@ jobs:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
services:
mysql:
image: mysql:5.7
@ -130,10 +133,9 @@ jobs:
run: |
pip-install
setup-mysql
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
env:
SUPERSET__SQLALCHEMY_DATABASE_URI: |
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
run: |
./scripts/python_tests.sh
- name: Upload code coverage
@ -149,6 +151,8 @@ jobs:
PYTHONPATH: ${{ github.workspace }}
SUPERSET_CONFIG: tests.superset_test_config
REDIS_PORT: 16379
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
services:
redis:
image: redis:5-alpine
@ -166,10 +170,9 @@ jobs:
run: |
pip-install
mkdir ${{ github.workspace }}/.temp
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
env:
SUPERSET__SQLALCHEMY_DATABASE_URI: |
sqlite:///${{ github.workspace }}/.temp/unittest.db
run: |
./scripts/python_tests.sh
- name: Upload code coverage

18
docker-compose.yml
View File

@ -83,12 +83,28 @@ services:
superset-worker:
build: *superset-build
container_name: superset_worker
command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair"]
command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair", "-l", "INFO"]
env_file: docker/.env
restart: unless-stopped
depends_on: *superset-depends-on
volumes: *superset-volumes
superset-tests-worker:
build: *superset-build
container_name: superset_tests_worker
command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair", "-l", "INFO"]
env_file: docker/.env
environment:
DATABASE_HOST: localhost
DATABASE_DB: test
REDIS_CELERY_DB: 2
REDIS_RESULTS_DB: 3
REDIS_HOST: localhost
network_mode: host
restart: unless-stopped
depends_on: *superset-depends-on
volumes: *superset-volumes
volumes:
superset_home:
external: false

8
docker/pythonpath_dev/superset_config.py
View File

@ -63,19 +63,23 @@ SQLALCHEMY_DATABASE_URI = "%s://%s:%s@%s:%s/%s" % (
REDIS_HOST = get_env_variable("REDIS_HOST")
REDIS_PORT = get_env_variable("REDIS_PORT")
REDIS_CELERY_DB = get_env_variable("REDIS_CELERY_DB", 0)
REDIS_RESULTS_DB = get_env_variable("REDIS_CELERY_DB", 1)
RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
class CeleryConfig(object):
BROKER_URL = "redis://%s:%s/0" % (REDIS_HOST, REDIS_PORT)
BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
CELERY_IMPORTS = ("superset.sql_lab",)
CELERY_RESULT_BACKEND = "redis://%s:%s/1" % (REDIS_HOST, REDIS_PORT)
CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}}
CELERY_TASK_PROTOCOL = 1
CELERY_CONFIG = CeleryConfig
SQLLAB_CTAS_NO_LIMIT = True
#
# Optionally import superset_config_docker.py (which will have been included on

10
scripts/tests/README.md
View File

@ -49,3 +49,13 @@ scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-i
```$bash
scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-reset-db
```
- Example for not running tests just initialize the test DB (drop/create, upgrade and load examples)
```$bash
scripts/tests/run.sh . --no-tests
```
- Example for just resetting the tests DB
```$bash
scripts/tests/run.sh . --reset-db
```

30
scripts/tests/run.sh
View File

@ -26,8 +26,19 @@ function reset_db() {
echo --------------------
echo Reseting test DB
echo --------------------
docker exec -i superset_db bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'DROP DATABASE IF EXISTS ${DB_NAME};'"
docker exec -i superset_db bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'CREATE DATABASE ${DB_NAME};'"
docker-compose stop superset-tests-worker
RESET_DB_CMD="psql \"postgresql://superset:superset@127.0.0.1:5432\" <<-EOF
DROP DATABASE IF EXISTS ${DB_NAME};
CREATE DATABASE ${DB_NAME};
\\c ${DB_NAME}
DROP SCHEMA IF EXISTS sqllab_test_db;
CREATE SCHEMA sqllab_test_db;
DROP SCHEMA IF EXISTS admin_database;
CREATE SCHEMA admin_database;
EOF
"
docker exec -i superset_db bash -c "${RESET_DB_CMD}"
docker-compose start superset-tests-worker
}
#
@ -71,6 +82,7 @@ export SUPERSET__SQLALCHEMY_DATABASE_URI=${SUPERSET__SQLALCHEMY_DATABASE_URI:-po
export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.superset_test_config}
RUN_INIT=1
RUN_RESET_DB=1
RUN_TESTS=1
TEST_MODULE="${1}"
# Shift to pass the first cmd parameter for the test module
@ -88,6 +100,15 @@ while (( "$#" )); do
RUN_RESET_DB=0
shift 1
;;
--no-tests)
RUN_TESTS=0
shift 1
;;
--reset-db)
RUN_TESTS=0
RUN_INIT=0
shift 1
;;
--) # end argument parsing
shift
break
@ -122,4 +143,7 @@ then
test_init
fi
nosetests --exclude=load_examples_test "${TEST_MODULE}"
if [ $RUN_TESTS -eq 1 ]
then
nosetests --exclude=load_examples_test "${TEST_MODULE}"
fi

29
tests/celery_tests.py
View File

@ -45,7 +45,8 @@ from superset.utils.core import get_example_database
from .base_tests import SupersetTestCase
CELERY_SHORT_SLEEP_TIME = 2
CELERY_SLEEP_TIME = 5
CELERY_SLEEP_TIME = 10
DROP_TABLE_SLEEP_TIME = 10
class UtilityFunctionTests(SupersetTestCase):
@ -115,32 +116,9 @@ class CeleryTestCase(SupersetTestCase):
@classmethod
def setUpClass(cls):
with app.app_context():
class CeleryConfig(object):
BROKER_URL = app.config["CELERY_CONFIG"].BROKER_URL
CELERY_IMPORTS = ("superset.sql_lab",)
CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}}
CONCURRENCY = 1
app.config["CELERY_CONFIG"] = CeleryConfig
db.session.query(Query).delete()
db.session.commit()
base_dir = app.config["BASE_DIR"]
worker_command = base_dir + "/bin/superset worker -w 2"
subprocess.Popen(worker_command, shell=True, stdout=subprocess.PIPE)
@classmethod
def tearDownClass(cls):
subprocess.call(
"ps auxww | grep 'celeryd' | awk '{print $2}' | xargs kill -9", shell=True
)
subprocess.call(
"ps auxww | grep 'superset worker' | awk '{print $2}' | xargs kill -9",
shell=True,
)
def run_sql(
self, db_id, sql, client_id=None, cta=False, tmp_table="tmp", async_=False
):
@ -286,7 +264,7 @@ class CeleryTestCase(SupersetTestCase):
table_name = "tmp_async_4"
self.drop_table_if_exists(table_name, main_db)
time.sleep(CELERY_SLEEP_TIME)
time.sleep(DROP_TABLE_SLEEP_TIME)
sql_where = "SELECT name FROM birth_names WHERE name='James' LIMIT 10"
result = self.run_sql(
@ -305,6 +283,7 @@ class CeleryTestCase(SupersetTestCase):
self.assertEqual(QueryStatus.SUCCESS, query.status)
self.assertTrue(f"FROM {table_name}" in query.select_sql)
self.assertEqual(
f"CREATE TABLE {table_name} AS \n"
"SELECT name FROM birth_names "

15
tests/superset_test_config.py
View File

@ -32,8 +32,8 @@ if "SUPERSET__SQLALCHEMY_DATABASE_URI" in os.environ:
if "sqlite" in SQLALCHEMY_DATABASE_URI:
logger.warning(
"SQLite Database support for metadata databases will be \
removed in a future version of Superset."
"SQLite Database support for metadata databases will be "
"removed in a future version of Superset."
)
SQL_MAX_ROW = 666
@ -57,11 +57,16 @@ ENABLE_ROW_LEVEL_SECURITY = True
CACHE_CONFIG = {"CACHE_TYPE": "simple"}
REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
REDIS_PORT = os.environ.get("REDIS_PORT", "6379")
REDIS_CELERY_DB = os.environ.get("REDIS_CELERY_DB", 2)
REDIS_RESULTS_DB = os.environ.get("REDIS_RESULTS_DB", 3)
class CeleryConfig(object):
BROKER_URL = "redis://{}:{}".format(
os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", "6379")
)
BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
CELERY_IMPORTS = ("superset.sql_lab",)
CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}}
CONCURRENCY = 1

13
tests/superset_test_config_thumbnails.py
View File

@ -56,9 +56,14 @@ EMAIL_NOTIFICATIONS = False
CACHE_CONFIG = {"CACHE_TYPE": "simple"}
REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
REDIS_PORT = os.environ.get("REDIS_PORT", "6379")
REDIS_CELERY_DB = os.environ.get("REDIS_CELERY_DB", 2)
REDIS_RESULTS_DB = os.environ.get("REDIS_RESULTS_DB", 3)
class CeleryConfig(object):
BROKER_URL = "redis://localhost"
BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks.thumbnails")
CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}}
CONCURRENCY = 1
@ -77,7 +82,11 @@ FEATURE_FLAGS = {
def init_thumbnail_cache(app: Flask) -> RedisCache:
return RedisCache(
host="localhost", key_prefix="superset_thumbnails_", default_timeout=10000
host=REDIS_HOST,
port=REDIS_PORT,
db=REDIS_CELERY_DB,
key_prefix="superset_thumbnails_",
default_timeout=10000,
)

46
tests/thumbnails_tests.py
View File

@ -38,49 +38,7 @@ from tests.test_app import app
from .base_tests import SupersetTestCase
class CeleryStartMixin:
@classmethod
def setUpClass(cls):
with app.app_context():
from cachelib.redis import RedisCache
class CeleryConfig(object):
BROKER_URL = "redis://localhost"
CELERY_IMPORTS = ("superset.tasks.thumbnails",)
CONCURRENCY = 1
app.config["CELERY_CONFIG"] = CeleryConfig
def init_thumbnail_cache(app) -> RedisCache:
return RedisCache(
host="localhost",
key_prefix="superset_thumbnails_",
default_timeout=10000,
)
app.config["THUMBNAIL_CACHE_CONFIG"] = init_thumbnail_cache
base_dir = app.config["BASE_DIR"]
worker_command = base_dir + "/bin/superset worker -w 2"
subprocess.Popen(
worker_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
@classmethod
def tearDownClass(cls):
subprocess.call(
"ps auxww | grep 'celeryd' | awk '{print $2}' | xargs kill -9", shell=True
)
subprocess.call(
"ps auxww | grep 'superset worker' | awk '{print $2}' | xargs kill -9",
shell=True,
)
class ThumbnailsSeleniumLive(CeleryStartMixin, LiveServerTestCase):
class ThumbnailsSeleniumLive(LiveServerTestCase):
def create_app(self):
return app
@ -108,7 +66,7 @@ class ThumbnailsSeleniumLive(CeleryStartMixin, LiveServerTestCase):
self.assertEqual(response.getcode(), 202)
class ThumbnailsTests(CeleryStartMixin, SupersetTestCase):
class ThumbnailsTests(SupersetTestCase):
mock_image = b"bytes mock image"

2
tox.ini
View File

@ -28,7 +28,7 @@ setenv =
SUPERSET_CONFIG = tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
py36-mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
py36-postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset
py36-postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test
py36-sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db
whitelist_externals =
npm