diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index f2caf2de38..655b5c85d5 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -150,7 +150,9 @@ jobs: SUPERSET_CONFIG: tests.integration_tests.superset_test_config REDIS_PORT: 16379 SUPERSET__SQLALCHEMY_DATABASE_URI: | - sqlite:///${{ github.workspace }}/.temp/unittest.db + sqlite:///${{ github.workspace }}/.temp/superset.db?check_same_thread=true + SUPERSET__SQLALCHEMY_EXAMPLES_URI: | + sqlite:///${{ github.workspace }}/.temp/examples.db?check_same_thread=true services: redis: image: redis:7-alpine diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py index d969895489..3530bdec1a 100644 --- a/tests/integration_tests/datasets/api_tests.py +++ b/tests/integration_tests/datasets/api_tests.py @@ -114,10 +114,6 @@ class TestDatasetApi(SupersetTestCase): @pytest.fixture() def create_virtual_datasets(self): with self.create_app().app_context(): - if backend() == "sqlite": - yield - return - datasets = [] admin = self.get_user("admin") main_db = get_main_database() @@ -140,10 +136,6 @@ class TestDatasetApi(SupersetTestCase): @pytest.fixture() def create_datasets(self): with self.create_app().app_context(): - if backend() == "sqlite": - yield - return - datasets = [] admin = self.get_user("admin") main_db = get_main_database() @@ -192,8 +184,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset list """ - if backend() == "sqlite": - return example_db = get_example_database() self.login(username="admin") @@ -232,8 +222,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset list gamma """ - if backend() == "sqlite": - return self.login(username="gamma") uri = "api/v1/dataset/" @@ -246,8 +234,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset list with database access """ - if backend() == "sqlite": - return self.login(username="gamma") @@ -289,8 +275,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset related databases gamma """ - if backend() == "sqlite": - return # Add main database access to gamma role main_db = get_main_database() @@ -320,8 +304,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset item """ - if backend() == "sqlite": - return table = self.get_energy_usage_dataset() main_db = get_main_database() @@ -385,8 +367,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset distinct schema """ - if backend() == "sqlite": - return def pg_test_query_parameter(query_parameter, expected_response): uri = f"api/v1/dataset/distinct/schema?q={prison.dumps(query_parameter)}" @@ -459,8 +439,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset distinct not allowed """ - if backend() == "sqlite": - return self.login(username="admin") uri = "api/v1/dataset/distinct/table_name" @@ -471,8 +449,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset distinct with gamma """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() @@ -491,8 +467,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset info """ - if backend() == "sqlite": - return self.login(username="admin") uri = "api/v1/dataset/_info" @@ -503,8 +477,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test info security """ - if backend() == "sqlite": - return self.login(username="admin") params = {"keys": ["permissions"]} @@ -525,8 +497,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset item """ - if backend() == "sqlite": - return main_db = get_main_database() self.login(username="admin") @@ -572,8 +542,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset item with column normalization enabled """ - if backend() == "sqlite": - return main_db = get_main_database() self.login(username="admin") @@ -601,8 +569,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset item gamma """ - if backend() == "sqlite": - return self.login(username="gamma") main_db = get_main_database() @@ -619,8 +585,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create item owner """ - if backend() == "sqlite": - return main_db = get_main_database() self.login(username="alpha") @@ -647,8 +611,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset item owner invalid """ - if backend() == "sqlite": - return admin = self.get_user("admin") main_db = get_main_database() @@ -671,8 +633,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset validate table uniqueness """ - if backend() == "sqlite": - return energy_usage_ds = self.get_energy_usage_dataset() self.login(username="admin") @@ -694,8 +654,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset with sql """ - if backend() == "sqlite": - return energy_usage_ds = self.get_energy_usage_dataset() self.login(username="admin") @@ -718,8 +676,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset with sql """ - if backend() == "sqlite": - return energy_usage_ds = self.get_energy_usage_dataset() self.login(username="alpha") @@ -777,8 +733,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset validate database exists """ - if backend() == "sqlite": - return self.login(username="admin") dataset_data = {"database": 1000, "schema": "", "table_name": "birth_names"} @@ -792,8 +746,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset validate table exists """ - if backend() == "sqlite": - return example_db = get_example_database() self.login(username="admin") @@ -820,8 +772,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset validate view exists """ - if backend() == "sqlite": - return mock_get_columns.return_value = [ { @@ -868,8 +818,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test create dataset sqlalchemy error """ - if backend() == "sqlite": - return mock_dao_create.side_effect = DAOCreateFailedError() self.login(username="admin") @@ -889,8 +837,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset item """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="admin") @@ -908,8 +854,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset with override columns """ - if backend() == "sqlite": - return # Add default dataset dataset = self.insert_default_dataset() @@ -947,8 +891,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset with override columns """ - if backend() == "sqlite": - return # Add default dataset main_db = get_main_database() @@ -997,8 +939,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset create column """ - if backend() == "sqlite": - return # create example dataset by Command dataset = self.insert_default_dataset() @@ -1095,8 +1035,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset delete column """ - if backend() == "sqlite": - return # create example dataset by Command dataset = self.insert_default_dataset() @@ -1147,8 +1085,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset columns """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() @@ -1186,8 +1122,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset delete metric """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() metrics_query = ( @@ -1232,8 +1166,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset columns uniqueness """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() @@ -1255,8 +1187,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset metric uniqueness """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() @@ -1278,8 +1208,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset columns duplicate """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() @@ -1306,8 +1234,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset metric duplicate """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() @@ -1334,8 +1260,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset item gamma """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="gamma") @@ -1350,8 +1274,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Tests that no username is returned """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="admin") @@ -1374,8 +1296,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Tests that no username is returned """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="admin") @@ -1397,8 +1317,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset item not owned """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="alpha") @@ -1413,8 +1331,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset item owner invalid """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="admin") @@ -1429,8 +1345,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset uniqueness """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="admin") @@ -1455,8 +1369,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test update dataset sqlalchemy error """ - if backend() == "sqlite": - return mock_dao_update.side_effect = DAOUpdateFailedError() @@ -1476,8 +1388,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset item """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() view_menu = security_manager.find_view_menu(dataset.get_perm()) @@ -1496,8 +1406,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete item not owned """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="alpha") @@ -1511,8 +1419,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete item not authorized """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="gamma") @@ -1527,8 +1433,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset sqlalchemy error """ - if backend() == "sqlite": - return mock_dao_delete.side_effect = DAODeleteFailedError() @@ -1547,8 +1451,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset column """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] column_id = dataset.columns[0].id @@ -1563,8 +1465,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset column not found """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] non_id = self.get_nonexistent_numeric_id(TableColumn) @@ -1587,8 +1487,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset column not owned """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] column_id = dataset.columns[0].id @@ -1604,8 +1502,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset column """ - if backend() == "sqlite": - return mock_dao_delete.side_effect = DAODeleteFailedError() dataset = self.get_fixture_datasets()[0] @@ -1622,8 +1518,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset metric """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] test_metric = SqlMetric( @@ -1643,8 +1537,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset metric not found """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] non_id = self.get_nonexistent_numeric_id(SqlMetric) @@ -1667,8 +1559,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset metric not owned """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] metric_id = dataset.metrics[0].id @@ -1684,8 +1574,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test delete dataset metric """ - if backend() == "sqlite": - return mock_dao_delete.side_effect = DAODeleteFailedError() dataset = self.get_fixture_datasets()[0] @@ -1702,8 +1590,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test bulk delete dataset items """ - if backend() == "sqlite": - return datasets = self.get_fixture_datasets() dataset_ids = [dataset.id for dataset in datasets] @@ -1734,8 +1620,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test bulk delete item not owned """ - if backend() == "sqlite": - return datasets = self.get_fixture_datasets() dataset_ids = [dataset.id for dataset in datasets] @@ -1750,8 +1634,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test bulk delete item not found """ - if backend() == "sqlite": - return datasets = self.get_fixture_datasets() dataset_ids = [dataset.id for dataset in datasets] @@ -1767,8 +1649,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test bulk delete item not authorized """ - if backend() == "sqlite": - return datasets = self.get_fixture_datasets() dataset_ids = [dataset.id for dataset in datasets] @@ -1783,8 +1663,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test bulk delete item incorrect request """ - if backend() == "sqlite": - return datasets = self.get_fixture_datasets() dataset_ids = [dataset.id for dataset in datasets] @@ -1799,8 +1677,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test item refresh """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() # delete a column @@ -1830,8 +1706,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test item refresh not found dataset """ - if backend() == "sqlite": - return max_id = db.session.query(func.max(SqlaTable.id)).scalar() @@ -1844,8 +1718,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test item refresh not owned dataset """ - if backend() == "sqlite": - return dataset = self.insert_default_dataset() self.login(username="alpha") @@ -1861,8 +1733,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test export dataset """ - if backend() == "sqlite": - return birth_names_dataset = self.get_birth_names_dataset() # TODO: fix test for presto @@ -1896,8 +1766,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test export dataset not found """ - if backend() == "sqlite": - return max_id = db.session.query(func.max(SqlaTable.id)).scalar() # Just one does not exist and we get 404 @@ -1912,8 +1780,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test export dataset has gamma """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] @@ -1943,8 +1809,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test export dataset """ - if backend() == "sqlite": - return birth_names_dataset = self.get_birth_names_dataset() # TODO: fix test for presto @@ -1967,8 +1831,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test export dataset not found """ - if backend() == "sqlite": - return # Just one does not exist and we get 404 argument = [-1, 1] @@ -1983,8 +1845,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test export dataset has gamma """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] @@ -2003,8 +1863,6 @@ class TestDatasetApi(SupersetTestCase): Dataset API: Test get chart and dashboard count related to a dataset :return: """ - if backend() == "sqlite": - return self.login(username="admin") table = self.get_birth_names_dataset() @@ -2019,8 +1877,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test related objects not found """ - if backend() == "sqlite": - return max_id = db.session.query(func.max(SqlaTable.id)).scalar() # id does not exist and we get 404 @@ -2042,8 +1898,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test custom dataset_is_null_or_empty filter for sql """ - if backend() == "sqlite": - return arguments = { "filters": [ @@ -2078,8 +1932,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test import dataset """ - if backend() == "sqlite": - return self.login(username="admin") uri = "api/v1/dataset/import/" @@ -2113,9 +1965,6 @@ class TestDatasetApi(SupersetTestCase): db.session.commit() def test_import_dataset_v0_export(self): - if backend() == "sqlite": - return - num_datasets = db.session.query(SqlaTable).count() self.login(username="admin") @@ -2146,8 +1995,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test import existing dataset """ - if backend() == "sqlite": - return self.login(username="admin") uri = "api/v1/dataset/import/" @@ -2217,8 +2064,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test import invalid dataset """ - if backend() == "sqlite": - return self.login(username="admin") uri = "api/v1/dataset/import/" @@ -2270,8 +2115,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test import invalid dataset """ - if backend() == "sqlite": - return self.login(username="admin") uri = "api/v1/dataset/import/" @@ -2318,8 +2161,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test custom dataset_is_certified filter """ - if backend() == "sqlite": - return table_w_certification = SqlaTable( table_name="foo", @@ -2351,8 +2192,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test duplicate virtual dataset """ - if backend() == "sqlite": - return dataset = self.get_fixture_virtual_datasets()[0] @@ -2379,8 +2218,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test duplicate physical dataset """ - if backend() == "sqlite": - return dataset = self.get_fixture_datasets()[0] @@ -2395,8 +2232,6 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test duplicate dataset with existing name """ - if backend() == "sqlite": - return dataset = self.get_fixture_virtual_datasets()[0] @@ -2507,9 +2342,10 @@ class TestDatasetApi(SupersetTestCase): assert table.normalize_columns is False db.session.delete(table) + db.session.commit() + with examples_db.get_sqla_engine_with_context() as engine: engine.execute("DROP TABLE test_create_sqla_table_api") - db.session.commit() @pytest.mark.usefixtures( "load_energy_table_with_slice", "load_birth_names_dashboard_with_slices" diff --git a/tox.ini b/tox.ini index be79ff4cf8..1218fb79a3 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,9 @@ setenv = SUPERSET_HOME = {envtmpdir} mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test - sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db + sqlite: + SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db + SUPERSET__SQLALCHEMY_EXAMPLES_URI = sqlite:////{envtmpdir}/examples.db mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 # docker run -p 8080:8080 --name presto starburstdata/presto mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default