chore: update fixtures scope that are loading data into the analytical db (#17780)

* Update db setup fixture scope

* Load data into db only once

* Update fixture scopes for data loading

* cleanup imports

* try import

* Update scope

* fix cleanup

* Import all fixtures

* Separate data log and metadata creation

* Retain table description

* fix dtype

Co-authored-by: Bogdan Kyryliuk <bogdankyryliuk@dropbox.com>

tests/integration_tests/access_tests.py

@@ -25,12 +25,15 @@ from sqlalchemy import inspect
 
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.test_app import app  # isort:skip
 from superset import db, security_manager

tests/integration_tests/base_api_tests.py

@@ -18,6 +18,7 @@
 import json
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 import pytest

tests/integration_tests/cache_tests.py

@@ -24,6 +24,7 @@ from superset.common.db_query_status import QueryStatus
 from superset.extensions import cache_manager
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 from .base_tests import SupersetTestCase

tests/integration_tests/celery_tests.py

@@ -25,6 +25,7 @@ import unittest.mock as mock
 from typing import Optional
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import pytest

tests/integration_tests/charts/api_tests.py

@@ -40,9 +40,11 @@ from tests.integration_tests.base_api_tests import ApiOwnersTestCaseMixin
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.importexport import (
     chart_config,
@@ -53,9 +55,11 @@ from tests.integration_tests.fixtures.importexport import (
 )
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_slice,
+    load_unicode_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.insert_chart_mixin import InsertChartMixin
 from tests.integration_tests.test_app import app

tests/integration_tests/charts/commands_tests.py

@@ -34,6 +34,7 @@ from superset.models.core import Database
 from superset.models.slice import Slice
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (

tests/integration_tests/charts/data/api_tests.py

@@ -32,6 +32,7 @@ from tests.integration_tests.base_tests import (
 from tests.integration_tests.annotation_layers.fixtures import create_annotation_layers
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.test_app import app
 

tests/integration_tests/charts/schema_tests.py

@@ -26,6 +26,7 @@ from superset.charts.schemas import ChartDataQueryContextSchema
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.query_context import get_query_context
 

tests/integration_tests/cli_tests.py

@@ -30,6 +30,7 @@ import superset.cli
 from superset import app
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 logger = logging.getLogger(__name__)

tests/integration_tests/core_tests.py

@@ -27,6 +27,7 @@ from typing import Dict, List
 from urllib.parse import quote
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import pytest
@@ -44,6 +45,7 @@ from superset.utils.core import get_example_database
 from tests.integration_tests.conftest import with_feature_flags
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.test_app import app
 import superset.views.utils
@@ -73,6 +75,7 @@ from superset.views.database.views import DatabaseView
 from .base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 logger = logging.getLogger(__name__)

tests/integration_tests/dashboard_tests.py

@@ -33,16 +33,20 @@ from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.public_role import public_role_like_gamma
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_position,
+    load_unicode_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 from .base_tests import SupersetTestCase

tests/integration_tests/dashboard_utils.py

@@ -29,35 +29,30 @@ from superset.models.slice import Slice
 from superset.utils.core import get_example_default_schema
 
 
-def create_table_for_dashboard(
+def get_table(
-    df: DataFrame,
+    table_name: str, database: Database, schema: Optional[str] = None,
+):
+    schema = schema or get_example_default_schema()
+    table_source = ConnectorRegistry.sources["table"]
+    return (
+        db.session.query(table_source)
+        .filter_by(database_id=database.id, schema=schema, table_name=table_name)
+        .one_or_none()
+    )
+
+
+def create_table_metadata(
     table_name: str,
     database: Database,
-    dtype: Dict[str, Any],
     table_description: str = "",
     fetch_values_predicate: Optional[str] = None,
     schema: Optional[str] = None,
 ) -> SqlaTable:
     schema = schema or get_example_default_schema()
 
-    df.to_sql(
+    table = get_table(table_name, database, schema)
-        table_name,
-        database.get_sqla_engine(),
-        if_exists="replace",
-        chunksize=500,
-        dtype=dtype,
-        index=False,
-        method="multi",
-        schema=schema,
-    )
-
-    table_source = ConnectorRegistry.sources["table"]
-    table = (
-        db.session.query(table_source)
-        .filter_by(database_id=database.id, schema=schema, table_name=table_name)
-        .one_or_none()
-    )
     if not table:
+        table_source = ConnectorRegistry.sources["table"]
         table = table_source(schema=schema, table_name=table_name)
     if fetch_values_predicate:
         table.fetch_values_predicate = fetch_values_predicate

tests/integration_tests/dashboards/api_tests.py

@@ -53,9 +53,11 @@ from tests.integration_tests.fixtures.importexport import (
 from tests.integration_tests.utils.get_dashboards import get_dashboards_ids
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 DASHBOARDS_FIXTURE_COUNT = 10

tests/integration_tests/dashboards/commands_tests.py

@@ -48,6 +48,7 @@ from tests.integration_tests.fixtures.importexport import (
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 

tests/integration_tests/dashboards/dao_tests.py

@@ -28,6 +28,7 @@ from superset.models.dashboard import Dashboard
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 

tests/integration_tests/dashboards/filter_state/api_tests.py

@@ -30,6 +30,7 @@ from superset.models.dashboard import Dashboard
 from tests.integration_tests.base_tests import login
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.test_app import app
 

tests/integration_tests/dashboards/security/security_dataset_tests.py

@@ -28,6 +28,7 @@ from tests.integration_tests.dashboards.consts import *
 from tests.integration_tests.dashboards.dashboard_test_utils import *
 from tests.integration_tests.dashboards.superset_factory_util import *
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 

tests/integration_tests/dashboards/security/security_rbac_tests.py

@@ -32,6 +32,7 @@ from tests.integration_tests.dashboards.superset_factory_util import (
 )
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.public_role import public_role_like_gamma
 from tests.integration_tests.fixtures.query_context import get_query_context

tests/integration_tests/databases/api_tests.py

@@ -47,13 +47,16 @@ from superset.utils.core import get_example_database, get_main_database
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.certificates import ssl_certificate
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.fixtures.importexport import (
     database_config,
@@ -63,6 +66,7 @@ from tests.integration_tests.fixtures.importexport import (
 )
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_position,
+    load_unicode_data,
 )
 from tests.integration_tests.test_app import app
 

tests/integration_tests/databases/commands_tests.py

@@ -44,8 +44,10 @@ from superset.utils.core import backend, get_example_database
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (

tests/integration_tests/datasets/api_tests.py

@@ -46,8 +46,10 @@ from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.conftest import CTAS_SCHEMA_NAME
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (

tests/integration_tests/datasets/commands_tests.py

@@ -33,6 +33,7 @@ from superset.models.core import Database
 from superset.utils.core import get_example_database, get_example_default_schema
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.energy_dashboard import (
+    load_energy_table_data,
     load_energy_table_with_slice,
 )
 from tests.integration_tests.fixtures.importexport import (
@@ -45,6 +46,7 @@ from tests.integration_tests.fixtures.importexport import (
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 

tests/integration_tests/datasource_tests.py

@@ -31,6 +31,7 @@ from superset.utils.core import get_example_database, get_example_default_schema
 from tests.integration_tests.base_tests import db_insert_temp_object, SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.datasource import get_datasource_post
 

tests/integration_tests/db_engine_specs/base_engine_spec_tests.py

@@ -35,8 +35,14 @@ from superset.utils.core import get_example_database
 from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
 from tests.integration_tests.test_app import app
 
-from ..fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices
+from ..fixtures.birth_names_dashboard import (
-from ..fixtures.energy_dashboard import load_energy_table_with_slice
+    load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
+)
+from ..fixtures.energy_dashboard import (
+    load_energy_table_data,
+    load_energy_table_with_slice,
+)
 from ..fixtures.pyodbcRow import Row
 
 

tests/integration_tests/db_engine_specs/bigquery_tests.py

@@ -29,6 +29,7 @@ from superset.sql_parse import Table
 from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 

tests/integration_tests/fixtures/__init__.py

@@ -19,7 +19,7 @@ from .birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
     load_birth_names_dashboard_with_slices_module_scope,
 )
-from .energy_dashboard import load_energy_table_with_slice
+from .energy_dashboard import load_energy_table_data, load_energy_table_with_slice
 from .public_role import public_role_like_gamma, public_role_like_test_role
 from .unicode_dashboard import (
     load_unicode_dashboard_with_position,

tests/integration_tests/fixtures/birth_names_dashboard.py

@@ -23,7 +23,7 @@ from typing import Any, Dict, List, Optional
 import pandas as pd
 import pytest
 from pandas import DataFrame
-from sqlalchemy import DateTime, String, TIMESTAMP
+from sqlalchemy import DateTime, String
 
 from superset import ConnectorRegistry, db
 from superset.connectors.sqla.models import SqlaTable
@@ -31,29 +31,14 @@ from superset.models.core import Database
 from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
 from superset.utils.core import get_example_database, get_example_default_schema
-from tests.integration_tests.dashboard_utils import create_table_for_dashboard
+from tests.integration_tests.dashboard_utils import create_table_metadata
 from tests.integration_tests.test_app import app
 
-
+BIRTH_NAMES_TBL_NAME = "birth_names"
-@pytest.fixture()
-def load_birth_names_dashboard_with_slices():
-    dash_id_to_delete, slices_ids_to_delete = _load_data()
-    yield
-    with app.app_context():
-        _cleanup(dash_id_to_delete, slices_ids_to_delete)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
-def load_birth_names_dashboard_with_slices_module_scope():
+def load_birth_names_data():
-    dash_id_to_delete, slices_ids_to_delete = _load_data()
-    yield
-    with app.app_context():
-        _cleanup(dash_id_to_delete, slices_ids_to_delete)
-
-
-def _load_data():
-    table_name = "birth_names"
-
     with app.app_context():
         database = get_example_database()
         df = _get_dataframe(database)
@@ -63,35 +48,61 @@ def _load_data():
             "state": String(10),
             "name": String(255),
         }
-        table = _create_table(
+
-            df=df,
+        df.to_sql(
-            table_name=table_name,
+            BIRTH_NAMES_TBL_NAME,
-            database=database,
+            database.get_sqla_engine(),
+            if_exists="replace",
+            chunksize=500,
             dtype=dtype,
-            fetch_values_predicate="123 = 123",
+            index=False,
+            method="multi",
+            schema=get_example_default_schema(),
         )
+    yield
+    with app.app_context():
+        engine = get_example_database().get_sqla_engine()
+        engine.execute("DROP TABLE IF EXISTS birth_names")
 
-        from superset.examples.birth_names import create_dashboard, create_slices
 
-        slices, _ = create_slices(table, admin_owner=False)
+@pytest.fixture()
-        dash = create_dashboard(slices)
+def load_birth_names_dashboard_with_slices(load_birth_names_data):
-        slices_ids_to_delete = [slice.id for slice in slices]
+    with app.app_context():
-        dash_id_to_delete = dash.id
+        dash_id_to_delete, slices_ids_to_delete = _create_dashboards()
-        return dash_id_to_delete, slices_ids_to_delete
+        yield
+        _cleanup(dash_id_to_delete, slices_ids_to_delete)
+
+
+@pytest.fixture(scope="module")
+def load_birth_names_dashboard_with_slices_module_scope(load_birth_names_data):
+    with app.app_context():
+        dash_id_to_delete, slices_ids_to_delete = _create_dashboards()
+        yield
+        _cleanup(dash_id_to_delete, slices_ids_to_delete)
+
+
+def _create_dashboards():
+    table = _create_table(
+        table_name=BIRTH_NAMES_TBL_NAME,
+        database=get_example_database(),
+        fetch_values_predicate="123 = 123",
+    )
+
+    from superset.examples.birth_names import create_dashboard, create_slices
+
+    slices, _ = create_slices(table, admin_owner=False)
+    dash = create_dashboard(slices)
+    slices_ids_to_delete = [slice.id for slice in slices]
+    dash_id_to_delete = dash.id
+    return dash_id_to_delete, slices_ids_to_delete
 
 
 def _create_table(
-    df: DataFrame,
+    table_name: str, database: "Database", fetch_values_predicate: Optional[str] = None,
-    table_name: str,
-    database: "Database",
-    dtype: Dict[str, Any],
-    fetch_values_predicate: Optional[str] = None,
 ):
-    table = create_table_for_dashboard(
+    table = create_table_metadata(
-        df=df,
         table_name=table_name,
         database=database,
-        dtype=dtype,
         fetch_values_predicate=fetch_values_predicate,
     )
     from superset.examples.birth_names import _add_table_metrics, _set_table_metadata
@@ -115,8 +126,6 @@ def _cleanup(dash_id: int, slices_ids: List[int]) -> None:
     columns = [column for column in datasource.columns]
     metrics = [metric for metric in datasource.metrics]
 
-    engine = get_example_database().get_sqla_engine()
-    engine.execute("DROP TABLE IF EXISTS birth_names")
     for column in columns:
         db.session.delete(column)
     for metric in metrics:

tests/integration_tests/fixtures/energy_dashboard.py

@@ -15,34 +15,50 @@
 # specific language governing permissions and limitations
 # under the License.
 import random
-import textwrap
 from typing import Dict, Set
 
 import pandas as pd
 import pytest
-from pandas import DataFrame
 from sqlalchemy import column, Float, String
 
 from superset import db
 from superset.connectors.sqla.models import SqlaTable, SqlMetric
-from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
-from superset.utils.core import get_example_database
+from superset.utils.core import get_example_database, get_example_default_schema
-from tests.integration_tests.dashboard_utils import (
+from tests.integration_tests.dashboard_utils import create_slice, create_table_metadata
-    create_slice,
-    create_table_for_dashboard,
-)
 from tests.integration_tests.test_app import app
 
 misc_dash_slices: Set[str] = set()
 
 
-@pytest.fixture()
+ENERGY_USAGE_TBL_NAME = "energy_usage"
-def load_energy_table_with_slice():
+
-    table_name = "energy_usage"
+
-    df = _get_dataframe()
+@pytest.fixture(scope="session")
+def load_energy_table_data():
     with app.app_context():
-        _create_energy_table(df, table_name)
+        database = get_example_database()
+        df = _get_dataframe()
+        df.to_sql(
+            ENERGY_USAGE_TBL_NAME,
+            database.get_sqla_engine(),
+            if_exists="replace",
+            chunksize=500,
+            index=False,
+            dtype={"source": String(255), "target": String(255), "value": Float()},
+            method="multi",
+            schema=get_example_default_schema(),
+        )
+    yield
+    with app.app_context():
+        engine = get_example_database().get_sqla_engine()
+        engine.execute("DROP TABLE IF EXISTS energy_usage")
+
+
+@pytest.fixture()
+def load_energy_table_with_slice(load_energy_table_data):
+    with app.app_context():
+        _create_energy_table()
         yield
         _cleanup()
 
@@ -52,13 +68,11 @@ def _get_dataframe():
     return pd.DataFrame.from_dict(data)
 
 
-def _create_energy_table(df: DataFrame, table_name: str):
+def _create_energy_table():
-    database = get_example_database()
+    table = create_table_metadata(
-
+        table_name=ENERGY_USAGE_TBL_NAME,
-    table_description = "Energy consumption"
+        database=get_example_database(),
-    schema = {"source": String(255), "target": String(255), "value": Float()}
+        table_description="Energy consumption",
-    table = create_table_for_dashboard(
-        df, table_name, database, schema, table_description
     )
     table.fetch_metadata()
 
@@ -96,8 +110,6 @@ def _create_and_commit_energy_slice(
 
 
 def _cleanup() -> None:
-    engine = get_example_database().get_sqla_engine()
-    engine.execute("DROP TABLE IF EXISTS energy_usage")
     for slice_data in _get_energy_slices():
         slice = (
             db.session.query(Slice)

tests/integration_tests/fixtures/unicode_dashboard.py

@@ -16,42 +16,58 @@
 # under the License.
 import pandas as pd
 import pytest
-from pandas import DataFrame
 from sqlalchemy import String
 
 from superset import db
 from superset.connectors.sqla.models import SqlaTable
 from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
-from superset.utils.core import get_example_database
+from superset.utils.core import get_example_database, get_example_default_schema
 from tests.integration_tests.dashboard_utils import (
     create_dashboard,
     create_slice,
-    create_table_for_dashboard,
+    create_table_metadata,
 )
 from tests.integration_tests.test_app import app
 
+UNICODE_TBL_NAME = "unicode_test"
+
+
+@pytest.fixture(scope="session")
+def load_unicode_data():
+    with app.app_context():
+        _get_dataframe().to_sql(
+            UNICODE_TBL_NAME,
+            get_example_database().get_sqla_engine(),
+            if_exists="replace",
+            chunksize=500,
+            dtype={"phrase": String(500)},
+            index=False,
+            method="multi",
+            schema=get_example_default_schema(),
+        )
+
+    yield
+    with app.app_context():
+        engine = get_example_database().get_sqla_engine()
+        engine.execute("DROP TABLE IF EXISTS unicode_test")
+
 
 @pytest.fixture()
-def load_unicode_dashboard_with_slice():
+def load_unicode_dashboard_with_slice(load_unicode_data):
-    table_name = "unicode_test"
     slice_name = "Unicode Cloud"
-    df = _get_dataframe()
     with app.app_context():
-        dash = _create_unicode_dashboard(df, table_name, slice_name, None)
+        dash = _create_unicode_dashboard(slice_name, None)
         yield
-
         _cleanup(dash, slice_name)
 
 
 @pytest.fixture()
-def load_unicode_dashboard_with_position():
+def load_unicode_dashboard_with_position(load_unicode_data):
-    table_name = "unicode_test"
     slice_name = "Unicode Cloud"
-    df = _get_dataframe()
     position = "{}"
     with app.app_context():
-        dash = _create_unicode_dashboard(df, table_name, slice_name, position)
+        dash = _create_unicode_dashboard(slice_name, position)
         yield
         _cleanup(dash, slice_name)
 
@@ -74,14 +90,8 @@ def _get_unicode_data():
     ]
 
 
-def _create_unicode_dashboard(
+def _create_unicode_dashboard(slice_title: str, position: str) -> Dashboard:
-    df: DataFrame, table_name: str, slice_title: str, position: str
+    table = create_table_metadata(UNICODE_TBL_NAME, get_example_database())
-) -> Dashboard:
-    database = get_example_database()
-    dtype = {
-        "phrase": String(500),
-    }
-    table = create_table_for_dashboard(df, table_name, database, dtype)
     table.fetch_metadata()
 
     if slice_title:
@@ -101,8 +111,6 @@ def _create_and_commit_unicode_slice(table: SqlaTable, title: str):
 
 
 def _cleanup(dash: Dashboard, slice_name: str) -> None:
-    engine = get_example_database().get_sqla_engine()
-    engine.execute("DROP TABLE IF EXISTS unicode_test")
     db.session.delete(dash)
     if slice_name:
         slice = db.session.query(Slice).filter_by(slice_name=slice_name).one_or_none()

tests/integration_tests/fixtures/world_bank_dashboard.py

@@ -22,7 +22,7 @@ from typing import Any, Dict, List
 import pandas as pd
 import pytest
 from pandas import DataFrame
-from sqlalchemy import DateTime, String, TIMESTAMP
+from sqlalchemy import DateTime, String
 
 from superset import db
 from superset.connectors.sqla.models import SqlaTable
@@ -32,43 +32,59 @@ from superset.models.slice import Slice
 from superset.utils.core import get_example_database, get_example_default_schema
 from tests.integration_tests.dashboard_utils import (
     create_dashboard,
-    create_table_for_dashboard,
+    create_table_metadata,
 )
 from tests.integration_tests.test_app import app
 
-
+WB_HEALTH_POPULATION = "wb_health_population"
-@pytest.fixture()
-def load_world_bank_dashboard_with_slices():
-    dash_id_to_delete, slices_ids_to_delete = _load_data()
-    yield
-    with app.app_context():
-        _cleanup(dash_id_to_delete, slices_ids_to_delete)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture(scope="session")
-def load_world_bank_dashboard_with_slices_module_scope():
+def load_world_bank_data():
-    dash_id_to_delete, slices_ids_to_delete = _load_data()
-    yield
-    with app.app_context():
-        _cleanup(dash_id_to_delete, slices_ids_to_delete)
-
-
-def _load_data():
-    table_name = "wb_health_population"
-
     with app.app_context():
         database = get_example_database()
-        schema = get_example_default_schema()
-        df = _get_dataframe(database)
         dtype = {
             "year": DateTime if database.backend != "presto" else String(255),
             "country_code": String(3),
             "country_name": String(255),
             "region": String(255),
         }
-        table = create_table_for_dashboard(
+        _get_dataframe(database).to_sql(
-            df, table_name, database, dtype, schema=schema
+            WB_HEALTH_POPULATION,
+            get_example_database().get_sqla_engine(),
+            if_exists="replace",
+            chunksize=500,
+            dtype=dtype,
+            index=False,
+            method="multi",
+            schema=get_example_default_schema(),
         )
+
+    yield
+    with app.app_context():
+        engine = get_example_database().get_sqla_engine()
+        engine.execute("DROP TABLE IF EXISTS wb_health_population")
+
+
+@pytest.fixture()
+def load_world_bank_dashboard_with_slices(load_world_bank_data):
+    with app.app_context():
+        dash_id_to_delete, slices_ids_to_delete = create_dashboard_for_loaded_data()
+        yield
+        _cleanup(dash_id_to_delete, slices_ids_to_delete)
+
+
+@pytest.fixture(scope="module")
+def load_world_bank_dashboard_with_slices_module_scope(load_world_bank_data):
+    with app.app_context():
+        dash_id_to_delete, slices_ids_to_delete = create_dashboard_for_loaded_data()
+        yield
+        _cleanup(dash_id_to_delete, slices_ids_to_delete)
+
+
+def create_dashboard_for_loaded_data():
+    with app.app_context():
+        table = create_table_metadata(WB_HEALTH_POPULATION, get_example_database())
         slices = _create_world_bank_slices(table)
         dash = _create_world_bank_dashboard(table, slices)
         slices_ids_to_delete = [slice.id for slice in slices]
@@ -112,8 +128,6 @@ def _create_world_bank_dashboard(table: SqlaTable, slices: List[Slice]) -> Dashb
 
 
 def _cleanup(dash_id: int, slices_ids: List[int]) -> None:
-    engine = get_example_database().get_sqla_engine()
-    engine.execute("DROP TABLE IF EXISTS wb_health_population")
     dash = db.session.query(Dashboard).filter_by(id=dash_id).first()
     db.session.delete(dash)
     for slice_id in slices_ids:

tests/integration_tests/import_export_tests.py

@@ -20,6 +20,7 @@ import json
 import unittest
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import pytest
@@ -28,6 +29,7 @@ from sqlalchemy.orm.session import make_transient
 
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.test_app import app
 from superset.dashboards.commands.importers.v0 import decode_dashboards
@@ -47,6 +49,7 @@ from superset.utils.core import get_example_database, get_example_default_schema
 
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from .base_tests import SupersetTestCase
 

tests/integration_tests/model_tests.py

@@ -22,6 +22,7 @@ from unittest import mock
 from superset.exceptions import SupersetException
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import pytest
@@ -38,7 +39,10 @@ from superset.models.sql_types.base import literal_dttm_type_factory
 from superset.utils.core import get_example_database
 
 from .base_tests import SupersetTestCase
-from .fixtures.energy_dashboard import load_energy_table_with_slice
+from .fixtures.energy_dashboard import (
+    load_energy_table_with_slice,
+    load_energy_table_data,
+)
 
 
 class TestDatabaseModel(SupersetTestCase):

tests/integration_tests/query_context_tests.py

@@ -34,6 +34,7 @@ from superset.utils.core import AdhocMetricExpressionType, backend, TimeRangeEnd
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.query_context import get_query_context
 

tests/integration_tests/reports/api_tests.py

@@ -43,6 +43,7 @@ from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.conftest import with_feature_flags
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.reports.utils import insert_report_schedule
 

tests/integration_tests/reports/commands_tests.py

@@ -59,9 +59,11 @@ from superset.reports.commands.log_prune import AsyncPruneReportScheduleLogComma
 from superset.utils.core import get_example_database
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices_module_scope,
+    load_world_bank_data,
 )
 from tests.integration_tests.reports.utils import insert_report_schedule
 from tests.integration_tests.test_app import app

tests/integration_tests/schedules_test.py

@@ -26,6 +26,7 @@ from slack import errors, WebClient
 
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 from tests.integration_tests.test_app import app
 from superset import db

tests/integration_tests/security_tests.py

@@ -48,9 +48,11 @@ from superset.views.access_requests import AccessRequestsModelView
 from .base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.energy_dashboard import (
     load_energy_table_with_slice,
+    load_energy_table_data,
 )
 from tests.integration_tests.fixtures.public_role import (
     public_role_like_gamma,
@@ -58,9 +60,11 @@ from tests.integration_tests.fixtures.public_role import (
 )
 from tests.integration_tests.fixtures.unicode_dashboard import (
     load_unicode_dashboard_with_slice,
+    load_unicode_data,
 )
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 NEW_SECURITY_CONVERGE_VIEWS = (

tests/integration_tests/sqla_models_tests.py

@@ -44,6 +44,7 @@ from superset.utils.core import (
 )
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 from .base_tests import SupersetTestCase

tests/integration_tests/sqllab_tests.py

@@ -58,6 +58,7 @@ from .base_tests import SupersetTestCase
 from .conftest import CTAS_SCHEMA_NAME
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 QUERY_1 = "SELECT * FROM birth_names LIMIT 1"

tests/integration_tests/strategy_tests.py

@@ -21,6 +21,7 @@ import json
 from unittest.mock import MagicMock
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 from sqlalchemy import String, Date, Float
@@ -42,8 +43,11 @@ from superset.tasks.cache import (
 )
 
 from .base_tests import SupersetTestCase
-from .dashboard_utils import create_dashboard, create_slice, create_table_for_dashboard
+from .dashboard_utils import create_dashboard, create_slice, create_table_metadata
-from .fixtures.unicode_dashboard import load_unicode_dashboard_with_slice
+from .fixtures.unicode_dashboard import (
+    load_unicode_dashboard_with_slice,
+    load_unicode_data,
+)
 
 URL_PREFIX = "http://0.0.0.0:8081"
 

tests/integration_tests/tasks/async_queries_tests.py

@@ -35,6 +35,7 @@ from superset.tasks.async_queries import (
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 from tests.integration_tests.fixtures.query_context import get_query_context
 from tests.integration_tests.test_app import app

tests/integration_tests/utils_tests.py

@@ -26,6 +26,7 @@ from typing import Any, Tuple, List, Optional
 from unittest.mock import Mock, patch
 from tests.integration_tests.fixtures.birth_names_dashboard import (
     load_birth_names_dashboard_with_slices,
+    load_birth_names_data,
 )
 
 import numpy as np
@@ -81,6 +82,7 @@ from superset.views.utils import (
 from tests.integration_tests.base_tests import SupersetTestCase
 from tests.integration_tests.fixtures.world_bank_dashboard import (
     load_world_bank_dashboard_with_slices,
+    load_world_bank_data,
 )
 
 from .fixtures.certificates import ssl_certificate

tox.ini

@@ -37,7 +37,7 @@ setenv =
     postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test
     sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db
     mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
-    # docker run -p 8080:8080 --name presto prestosql/presto
+    # docker run -p 8080:8080 --name presto starburstdata/presto
     mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default
     # based on https://github.com/big-data-europe/docker-hadoop
     # clone the repo & run docker-compose up -d to test locally