diff --git a/superset/cli.py b/superset/cli.py index 0382054daa..c97bbb6b42 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -168,7 +168,7 @@ def load_examples_run( examples.load_big_data() # load examples that are stored as YAML config files - examples.load_from_configs(force, load_test_data) + examples.load_examples_from_configs(force, load_test_data) @with_appcontext @@ -187,10 +187,28 @@ def load_examples( only_metadata: bool = False, force: bool = False, ) -> None: - """Loads a set of Slices and Dashboards and a supporting dataset """ + """Loads a set of Slices and Dashboards and a supporting dataset""" load_examples_run(load_test_data, load_big_data, only_metadata, force) +@with_appcontext +@superset.command() +@click.argument("directory") +@click.option( + "--overwrite", "-o", is_flag=True, help="Overwriting existing metadata definitions" +) +@click.option( + "--force", "-f", is_flag=True, help="Force load data even if table already exists" +) +def import_directory(directory: str, overwrite: bool, force: bool) -> None: + """Imports configs from a given directory""" + from superset.examples.utils import load_configs_from_directory + + load_configs_from_directory( + root=Path(directory), overwrite=overwrite, force_data=force, + ) + + @with_appcontext @superset.command() @click.option("--database_name", "-d", help="Database name to change") diff --git a/superset/commands/importers/v1/examples.py b/superset/commands/importers/v1/examples.py index 4bfab9f9cb..03ffd2c5dc 100644 --- a/superset/commands/importers/v1/examples.py +++ b/superset/commands/importers/v1/examples.py @@ -14,8 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=protected-access -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Set, Tuple from marshmallow import Schema from sqlalchemy.orm import Session @@ -23,19 +24,23 @@ from sqlalchemy.orm.exc import MultipleResultsFound from sqlalchemy.sql import select from superset import db +from superset.charts.commands.importers.v1 import ImportChartsCommand from superset.charts.commands.importers.v1.utils import import_chart from superset.charts.schemas import ImportV1ChartSchema from superset.commands.exceptions import CommandException from superset.commands.importers.v1 import ImportModelsCommand from superset.dao.base import BaseDAO +from superset.dashboards.commands.importers.v1 import ImportDashboardsCommand from superset.dashboards.commands.importers.v1.utils import ( find_chart_uuids, import_dashboard, update_id_refs, ) from superset.dashboards.schemas import ImportV1DashboardSchema +from superset.databases.commands.importers.v1 import ImportDatabasesCommand from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema +from superset.datasets.commands.importers.v1 import ImportDatasetsCommand from superset.datasets.commands.importers.v1.utils import import_dataset from superset.datasets.schemas import ImportV1DatasetSchema from superset.models.core import Database @@ -71,6 +76,15 @@ class ImportExamplesCommand(ImportModelsCommand): db.session.rollback() raise self.import_error() + @classmethod + def _get_uuids(cls) -> Set[str]: + return ( + ImportDatabasesCommand._get_uuids() + | ImportDatasetsCommand._get_uuids() + | ImportChartsCommand._get_uuids() + | ImportDashboardsCommand._get_uuids() + ) + # pylint: disable=too-many-locals, arguments-differ, too-many-branches @staticmethod def _import( diff --git a/superset/dashboards/commands/importers/v1/utils.py b/superset/dashboards/commands/importers/v1/utils.py index c7bd15e015..6eda781517 100644 --- a/superset/dashboards/commands/importers/v1/utils.py +++ b/superset/dashboards/commands/importers/v1/utils.py @@ -119,7 +119,7 @@ def update_id_refs( child["meta"]["chartId"] = chart_ids[child["meta"]["uuid"]] # fix native filter references - native_filter_configuration = fixed["metadata"].get( + native_filter_configuration = fixed.get("metadata", {}).get( "native_filter_configuration", [] ) for native_filter in native_filter_configuration: diff --git a/superset/datasets/commands/importers/v1/utils.py b/superset/datasets/commands/importers/v1/utils.py index 6b4dbeb7b8..d61be518a2 100644 --- a/superset/datasets/commands/importers/v1/utils.py +++ b/superset/datasets/commands/importers/v1/utils.py @@ -119,10 +119,12 @@ def import_dataset( example_database = get_example_database() try: table_exists = example_database.has_table_by_name(dataset.table_name) - except Exception as ex: + except Exception: # pylint: disable=broad-except # MySQL doesn't play nice with GSheets table names - logger.warning("Couldn't check if table %s exists, stopping import") - raise ex + logger.warning( + "Couldn't check if table %s exists, assuming it does", dataset.table_name + ) + table_exists = True if data_uri and (not table_exists or force_data): load_data(data_uri, dataset, example_database, session) diff --git a/superset/examples/__init__.py b/superset/examples/__init__.py index 161a52f4b4..a7742b0ef7 100644 --- a/superset/examples/__init__.py +++ b/superset/examples/__init__.py @@ -30,5 +30,5 @@ from .paris import load_paris_iris_geojson from .random_time_series import load_random_time_series_data from .sf_population_polygons import load_sf_population_polygons from .tabbed_dashboard import load_tabbed_dashboard -from .utils import load_from_configs +from .utils import load_examples_from_configs from .world_bank import load_world_bank_health_n_pop diff --git a/superset/examples/utils.py b/superset/examples/utils.py index 66ca811df2..fc5b78a230 100644 --- a/superset/examples/utils.py +++ b/superset/examples/utils.py @@ -14,18 +14,29 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import logging import re from pathlib import Path from typing import Any, Dict +import yaml from pkg_resources import resource_isdir, resource_listdir, resource_stream +from superset.commands.exceptions import CommandInvalidError from superset.commands.importers.v1.examples import ImportExamplesCommand +from superset.commands.importers.v1.utils import METADATA_FILE_NAME + +_logger = logging.getLogger(__name__) YAML_EXTENSIONS = {".yaml", ".yml"} -def load_from_configs(force_data: bool = False, load_test_data: bool = False) -> None: +def load_examples_from_configs( + force_data: bool = False, load_test_data: bool = False +) -> None: + """ + Load all the examples inside superset/examples/configs/. + """ contents = load_contents(load_test_data) command = ImportExamplesCommand(contents, overwrite=True, force_data=force_data) command.run() @@ -55,3 +66,35 @@ def load_contents(load_test_data: bool = False) -> Dict[str, Any]: ) return {str(path.relative_to(root)): content for path, content in contents.items()} + + +def load_configs_from_directory( + root: Path, overwrite: bool = True, force_data: bool = False +) -> None: + """ + Load all the examples from a given directory. + """ + contents: Dict[str, str] = {} + queue = [root] + while queue: + path_name = queue.pop() + if path_name.is_dir(): + queue.extend(path_name.glob("*")) + elif path_name.suffix.lower() in YAML_EXTENSIONS: + with open(path_name) as fp: + contents[str(path_name.relative_to(root))] = fp.read() + + # removing "type" from the metadata allows us to import any exported model + # from the unzipped directory directly + metadata = yaml.load(contents.get(METADATA_FILE_NAME, "{}")) + if "type" in metadata: + del metadata["type"] + contents[METADATA_FILE_NAME] = yaml.dump(metadata) + + command = ImportExamplesCommand( + contents, overwrite=overwrite, force_data=force_data + ) + try: + command.run() + except CommandInvalidError as ex: + _logger.error("An error occurred: %s", ex.normalized_messages())