diff --git a/superset/charts/commands/exceptions.py b/superset/charts/commands/exceptions.py index 2308d62a77..dd146d790b 100644 --- a/superset/charts/commands/exceptions.py +++ b/superset/charts/commands/exceptions.py @@ -23,6 +23,7 @@ from superset.commands.exceptions import ( CreateFailedError, DeleteFailedError, ForbiddenError, + ImportFailedError, UpdateFailedError, ) @@ -83,3 +84,7 @@ class ChartForbiddenError(ForbiddenError): class ChartBulkDeleteFailedError(CreateFailedError): message = _("Charts could not be deleted.") + + +class ChartImportError(ImportFailedError): + message = _("Import chart failed for an unknown reason") diff --git a/superset/charts/commands/importers/dispatcher.py b/superset/charts/commands/importers/dispatcher.py index e7f01491d7..2098ea7a6e 100644 --- a/superset/charts/commands/importers/dispatcher.py +++ b/superset/charts/commands/importers/dispatcher.py @@ -53,8 +53,7 @@ class ImportChartsCommand(BaseCommand): command.run() return except IncorrectVersionError: - # file is not handled by command, skip - pass + logger.debug("File not handled by command, skipping") except (CommandInvalidError, ValidationError) as exc: # found right version, but file is invalid logger.info("Command failed validation") diff --git a/superset/charts/commands/importers/v1/__init__.py b/superset/charts/commands/importers/v1/__init__.py index 086a37070f..4aed3fa341 100644 --- a/superset/charts/commands/importers/v1/__init__.py +++ b/superset/charts/commands/importers/v1/__init__.py @@ -22,6 +22,7 @@ from marshmallow.exceptions import ValidationError from sqlalchemy.orm import Session from superset import db +from superset.charts.commands.exceptions import ChartImportError from superset.charts.commands.importers.v1.utils import import_chart from superset.charts.schemas import ImportV1ChartSchema from superset.commands.base import BaseCommand @@ -105,9 +106,9 @@ class ImportChartsCommand(BaseCommand): try: self._import_bundle(db.session) db.session.commit() - except Exception as exc: + except Exception: db.session.rollback() - raise exc + raise ChartImportError() def validate(self) -> None: exceptions: List[ValidationError] = [] diff --git a/superset/commands/exceptions.py b/superset/commands/exceptions.py index 8f6b4c8e08..0eb5dba6d0 100644 --- a/superset/commands/exceptions.py +++ b/superset/commands/exceptions.py @@ -73,6 +73,11 @@ class ForbiddenError(CommandException): message = "Action is forbidden" +class ImportFailedError(CommandException): + status = 500 + message = "Import failed for an unknown reason" + + class OwnersNotFoundValidationError(ValidationError): status = 422 diff --git a/superset/dashboards/api.py b/superset/dashboards/api.py index 815bfd19fc..f16f358126 100644 --- a/superset/dashboards/api.py +++ b/superset/dashboards/api.py @@ -29,6 +29,7 @@ from werkzeug.wrappers import Response as WerkzeugResponse from werkzeug.wsgi import FileWrapper from superset import is_feature_enabled, thumbnail_cache +from superset.commands.exceptions import CommandInvalidError from superset.constants import RouteMethod from superset.dashboards.commands.bulk_delete import BulkDeleteDashboardCommand from superset.dashboards.commands.create import CreateDashboardCommand @@ -38,11 +39,13 @@ from superset.dashboards.commands.exceptions import ( DashboardCreateFailedError, DashboardDeleteFailedError, DashboardForbiddenError, + DashboardImportError, DashboardInvalidError, DashboardNotFoundError, DashboardUpdateFailedError, ) from superset.dashboards.commands.export import ExportDashboardsCommand +from superset.dashboards.commands.importers.dispatcher import ImportDashboardsCommand from superset.dashboards.commands.update import UpdateDashboardCommand from superset.dashboards.dao import DashboardDAO from superset.dashboards.filters import ( @@ -79,6 +82,7 @@ class DashboardRestApi(BaseSupersetModelRestApi): datamodel = SQLAInterface(Dashboard) include_route_methods = RouteMethod.REST_MODEL_VIEW_CRUD_SET | { RouteMethod.EXPORT, + RouteMethod.IMPORT, RouteMethod.RELATED, "bulk_delete", # not using RouteMethod since locally defined "favorite_status", @@ -641,3 +645,56 @@ class DashboardRestApi(BaseSupersetModelRestApi): for request_id in requested_ids ] return self.response(200, result=res) + + @expose("/import/", methods=["POST"]) + @protect() + @safe + @statsd_metrics + def import_(self) -> Response: + """Import dashboard(s) with associated charts/datasets/databases + --- + post: + requestBody: + content: + application/zip: + schema: + type: string + format: binary + responses: + 200: + description: Dashboard import result + content: + application/json: + schema: + type: object + properties: + message: + type: string + 400: + $ref: '#/components/responses/400' + 401: + $ref: '#/components/responses/401' + 422: + $ref: '#/components/responses/422' + 500: + $ref: '#/components/responses/500' + """ + upload = request.files.get("file") + if not upload: + return self.response_400() + with ZipFile(upload) as bundle: + contents = { + file_name: bundle.read(file_name).decode() + for file_name in bundle.namelist() + } + + command = ImportDashboardsCommand(contents) + try: + command.run() + return self.response(200, message="OK") + except CommandInvalidError as exc: + logger.warning("Import dashboard failed") + return self.response_422(message=exc.normalized_messages()) + except DashboardImportError as exc: + logger.exception("Import dashboard failed") + return self.response_500(message=str(exc)) diff --git a/superset/dashboards/commands/exceptions.py b/superset/dashboards/commands/exceptions.py index c6f78ee8b7..cc645da4bd 100644 --- a/superset/dashboards/commands/exceptions.py +++ b/superset/dashboards/commands/exceptions.py @@ -23,6 +23,7 @@ from superset.commands.exceptions import ( CreateFailedError, DeleteFailedError, ForbiddenError, + ImportFailedError, UpdateFailedError, ) @@ -62,3 +63,7 @@ class DashboardDeleteFailedError(DeleteFailedError): class DashboardForbiddenError(ForbiddenError): message = _("Changing this Dashboard is forbidden") + + +class DashboardImportError(ImportFailedError): + message = _("Import dashboard failed for an unknown reason") diff --git a/superset/dashboards/commands/importers/dispatcher.py b/superset/dashboards/commands/importers/dispatcher.py new file mode 100644 index 0000000000..91811c90fd --- /dev/null +++ b/superset/dashboards/commands/importers/dispatcher.py @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +from typing import Any, Dict + +from marshmallow.exceptions import ValidationError + +from superset.commands.base import BaseCommand +from superset.commands.exceptions import CommandInvalidError +from superset.commands.importers.exceptions import IncorrectVersionError +from superset.dashboards.commands.importers import v0, v1 + +logger = logging.getLogger(__name__) + +# list of different import formats supported; v0 should be last because +# the files are not versioned +command_versions = [ + v1.ImportDashboardsCommand, + v0.ImportDashboardsCommand, +] + + +class ImportDashboardsCommand(BaseCommand): + """ + Import dashboards. + + This command dispatches the import to different versions of the command + until it finds one that matches. + """ + + # pylint: disable=unused-argument + def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any): + self.contents = contents + + def run(self) -> None: + # iterate over all commands until we find a version that can + # handle the contents + for version in command_versions: + command = version(self.contents) + try: + command.run() + return + except IncorrectVersionError: + logger.debug("File not handled by command, skipping") + except (CommandInvalidError, ValidationError) as exc: + # found right version, but file is invalid + logger.info("Command failed validation") + raise exc + except Exception as exc: + # validation succeeded but something went wrong + logger.exception("Error running import command") + raise exc + + raise CommandInvalidError("Could not find a valid command to import file") + + def validate(self) -> None: + pass diff --git a/superset/dashboards/commands/importers/v1/__init__.py b/superset/dashboards/commands/importers/v1/__init__.py index e663e2e9c3..086fa0318d 100644 --- a/superset/dashboards/commands/importers/v1/__init__.py +++ b/superset/dashboards/commands/importers/v1/__init__.py @@ -32,6 +32,7 @@ from superset.commands.importers.v1.utils import ( load_yaml, METADATA_FILE_NAME, ) +from superset.dashboards.commands.exceptions import DashboardImportError from superset.dashboards.commands.importers.v1.utils import import_dashboard from superset.dashboards.schemas import ImportV1DashboardSchema from superset.databases.commands.importers.v1.utils import import_database @@ -154,9 +155,9 @@ class ImportDashboardsCommand(BaseCommand): try: self._import_bundle(db.session) db.session.commit() - except Exception as exc: + except Exception: db.session.rollback() - raise exc + raise DashboardImportError() def validate(self) -> None: exceptions: List[ValidationError] = [] diff --git a/superset/databases/api.py b/superset/databases/api.py index 4d8a0d4134..aa11a62c8e 100644 --- a/superset/databases/api.py +++ b/superset/databases/api.py @@ -44,6 +44,7 @@ from superset.databases.commands.exceptions import ( DatabaseCreateFailedError, DatabaseDeleteDatasetsExistFailedError, DatabaseDeleteFailedError, + DatabaseImportError, DatabaseInvalidError, DatabaseNotFoundError, DatabaseSecurityUnsafeError, @@ -775,6 +776,6 @@ class DatabaseRestApi(BaseSupersetModelRestApi): except CommandInvalidError as exc: logger.warning("Import database failed") return self.response_422(message=exc.normalized_messages()) - except Exception as exc: # pylint: disable=broad-except + except DatabaseImportError as exc: logger.exception("Import database failed") return self.response_500(message=str(exc)) diff --git a/superset/databases/commands/exceptions.py b/superset/databases/commands/exceptions.py index 1ef0a79442..48c63a2c97 100644 --- a/superset/databases/commands/exceptions.py +++ b/superset/databases/commands/exceptions.py @@ -22,6 +22,7 @@ from superset.commands.exceptions import ( CommandInvalidError, CreateFailedError, DeleteFailedError, + ImportFailedError, UpdateFailedError, ) from superset.security.analytics_db_safety import DBSecurityException @@ -114,3 +115,7 @@ class DatabaseDeleteFailedError(DeleteFailedError): class DatabaseSecurityUnsafeError(DBSecurityException): message = _("Stopped an unsafe database connection") + + +class DatabaseImportError(ImportFailedError): + message = _("Import database failed for an unknown reason") diff --git a/superset/databases/commands/importers/dispatcher.py b/superset/databases/commands/importers/dispatcher.py index a29f5ca108..fd02359176 100644 --- a/superset/databases/commands/importers/dispatcher.py +++ b/superset/databases/commands/importers/dispatcher.py @@ -51,8 +51,7 @@ class ImportDatabasesCommand(BaseCommand): command.run() return except IncorrectVersionError: - # file is not handled by this command, skip - pass + logger.debug("File not handled by command, skipping") except (CommandInvalidError, ValidationError) as exc: # found right version, but file is invalid logger.info("Command failed validation") diff --git a/superset/databases/commands/importers/v1/__init__.py b/superset/databases/commands/importers/v1/__init__.py index e0cc0f387d..c51ce3e3aa 100644 --- a/superset/databases/commands/importers/v1/__init__.py +++ b/superset/databases/commands/importers/v1/__init__.py @@ -29,6 +29,7 @@ from superset.commands.importers.v1.utils import ( load_yaml, METADATA_FILE_NAME, ) +from superset.databases.commands.exceptions import DatabaseImportError from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema from superset.datasets.commands.importers.v1.utils import import_dataset @@ -75,9 +76,9 @@ class ImportDatabasesCommand(BaseCommand): try: self._import_bundle(db.session) db.session.commit() - except Exception as exc: + except Exception: db.session.rollback() - raise exc + raise DatabaseImportError() def validate(self) -> None: exceptions: List[ValidationError] = [] diff --git a/superset/datasets/api.py b/superset/datasets/api.py index 1decc707ac..3609f8fdfb 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -40,6 +40,7 @@ from superset.datasets.commands.exceptions import ( DatasetCreateFailedError, DatasetDeleteFailedError, DatasetForbiddenError, + DatasetImportError, DatasetInvalidError, DatasetNotFoundError, DatasetRefreshFailedError, @@ -598,7 +599,7 @@ class DatasetRestApi(BaseSupersetModelRestApi): @safe @statsd_metrics def import_(self) -> Response: - """Import dataset (s) with associated databases + """Import dataset(s) with associated databases --- post: requestBody: @@ -642,6 +643,6 @@ class DatasetRestApi(BaseSupersetModelRestApi): except CommandInvalidError as exc: logger.warning("Import dataset failed") return self.response_422(message=exc.normalized_messages()) - except Exception as exc: # pylint: disable=broad-except + except DatasetImportError as exc: logger.exception("Import dataset failed") return self.response_500(message=str(exc)) diff --git a/superset/datasets/commands/exceptions.py b/superset/datasets/commands/exceptions.py index ee9d781e80..03071cac63 100644 --- a/superset/datasets/commands/exceptions.py +++ b/superset/datasets/commands/exceptions.py @@ -23,6 +23,7 @@ from superset.commands.exceptions import ( CreateFailedError, DeleteFailedError, ForbiddenError, + ImportFailedError, UpdateFailedError, ) from superset.views.base import get_datasource_exist_error_msg @@ -170,3 +171,7 @@ class DatasetRefreshFailedError(UpdateFailedError): class DatasetForbiddenError(ForbiddenError): message = _("Changing this dataset is forbidden") + + +class DatasetImportError(ImportFailedError): + message = _("Import dataset failed for an unknown reason") diff --git a/superset/datasets/commands/importers/dispatcher.py b/superset/datasets/commands/importers/dispatcher.py index 99a4c269e7..b268463331 100644 --- a/superset/datasets/commands/importers/dispatcher.py +++ b/superset/datasets/commands/importers/dispatcher.py @@ -56,8 +56,7 @@ class ImportDatasetsCommand(BaseCommand): command.run() return except IncorrectVersionError: - # file is not handled by command, skip - pass + logger.debug("File not handled by command, skipping") except (CommandInvalidError, ValidationError) as exc: # found right version, but file is invalid logger.info("Command failed validation") diff --git a/superset/datasets/commands/importers/v1/__init__.py b/superset/datasets/commands/importers/v1/__init__.py index 1b2609110e..43ea458abf 100644 --- a/superset/datasets/commands/importers/v1/__init__.py +++ b/superset/datasets/commands/importers/v1/__init__.py @@ -32,6 +32,7 @@ from superset.commands.importers.v1.utils import ( from superset.connectors.sqla.models import SqlaTable from superset.databases.commands.importers.v1.utils import import_database from superset.databases.schemas import ImportV1DatabaseSchema +from superset.datasets.commands.exceptions import DatasetImportError from superset.datasets.commands.importers.v1.utils import import_dataset from superset.datasets.schemas import ImportV1DatasetSchema @@ -80,9 +81,9 @@ class ImportDatasetsCommand(BaseCommand): try: self._import_bundle(db.session) db.session.commit() - except Exception as exc: + except Exception: db.session.rollback() - raise exc + raise DatasetImportError() def validate(self) -> None: exceptions: List[ValidationError] = [] diff --git a/tests/dashboards/api_tests.py b/tests/dashboards/api_tests.py index 162ea22884..72aee218e6 100644 --- a/tests/dashboards/api_tests.py +++ b/tests/dashboards/api_tests.py @@ -15,18 +15,19 @@ # specific language governing permissions and limitations # under the License. # isort:skip_file +# pylint: disable=too-many-public-methods, no-self-use, invalid-name, too-many-arguments """Unit tests for Superset""" import json from io import BytesIO from typing import List, Optional from unittest.mock import patch -from zipfile import is_zipfile +from zipfile import is_zipfile, ZipFile import pytest import prison +import yaml from sqlalchemy.sql import func -import tests.test_app from freezegun import freeze_time from sqlalchemy import and_ from superset import db, security_manager @@ -37,6 +38,14 @@ from superset.views.base import generate_download_headers from tests.base_api_tests import ApiOwnersTestCaseMixin from tests.base_tests import SupersetTestCase +from tests.fixtures.importexport import ( + chart_config, + database_config, + dashboard_config, + dashboard_metadata_config, + dataset_config, + dataset_metadata_config, +) DASHBOARDS_FIXTURE_COUNT = 10 @@ -142,7 +151,7 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin): ], "position_json": "", "published": False, - "url": f"/superset/dashboard/slug1/", + "url": "/superset/dashboard/slug1/", "slug": "slug1", "table_names": "", "thumbnail_url": dashboard.thumbnail_url, @@ -162,7 +171,7 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin): Dashboard API: Test info """ self.login(username="admin") - uri = f"api/v1/dashboard/_info" + uri = "api/v1/dashboard/_info" rv = self.get_assert_metric(uri, "info") self.assertEqual(rv.status_code, 200) @@ -1077,3 +1086,86 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin): db.session.delete(dashboard) db.session.commit() + + def test_import_dashboard(self): + """ + Dashboard API: Test import dashboard + """ + self.login(username="admin") + uri = "api/v1/dashboard/import/" + + buf = BytesIO() + with ZipFile(buf, "w") as bundle: + with bundle.open("metadata.yaml", "w") as fp: + fp.write(yaml.safe_dump(dashboard_metadata_config).encode()) + with bundle.open("databases/imported_database.yaml", "w") as fp: + fp.write(yaml.safe_dump(database_config).encode()) + with bundle.open("datasets/imported_dataset.yaml", "w") as fp: + fp.write(yaml.safe_dump(dataset_config).encode()) + with bundle.open("charts/imported_chart.yaml", "w") as fp: + fp.write(yaml.safe_dump(chart_config).encode()) + with bundle.open("dashboards/imported_dashboard.yaml", "w") as fp: + fp.write(yaml.safe_dump(dashboard_config).encode()) + buf.seek(0) + + form_data = { + "file": (buf, "dashboard_export.zip"), + } + rv = self.client.post(uri, data=form_data, content_type="multipart/form-data") + response = json.loads(rv.data.decode("utf-8")) + + assert rv.status_code == 200 + assert response == {"message": "OK"} + + dashboard = ( + db.session.query(Dashboard).filter_by(uuid=dashboard_config["uuid"]).one() + ) + assert dashboard.dashboard_title == "Test dash" + + assert len(dashboard.slices) == 1 + chart = dashboard.slices[0] + assert str(chart.uuid) == chart_config["uuid"] + + dataset = chart.table + assert str(dataset.uuid) == dataset_config["uuid"] + + database = dataset.database + assert str(database.uuid) == database_config["uuid"] + + db.session.delete(dashboard) + db.session.delete(chart) + db.session.delete(dataset) + db.session.delete(database) + db.session.commit() + + def test_import_dashboard_invalid(self): + """ + Dataset API: Test import invalid dashboard + """ + self.login(username="admin") + uri = "api/v1/dashboard/import/" + + buf = BytesIO() + with ZipFile(buf, "w") as bundle: + with bundle.open("metadata.yaml", "w") as fp: + fp.write(yaml.safe_dump(dataset_metadata_config).encode()) + with bundle.open("databases/imported_database.yaml", "w") as fp: + fp.write(yaml.safe_dump(database_config).encode()) + with bundle.open("datasets/imported_dataset.yaml", "w") as fp: + fp.write(yaml.safe_dump(dataset_config).encode()) + with bundle.open("charts/imported_chart.yaml", "w") as fp: + fp.write(yaml.safe_dump(chart_config).encode()) + with bundle.open("dashboards/imported_dashboard.yaml", "w") as fp: + fp.write(yaml.safe_dump(dashboard_config).encode()) + buf.seek(0) + + form_data = { + "file": (buf, "dashboard_export.zip"), + } + rv = self.client.post(uri, data=form_data, content_type="multipart/form-data") + response = json.loads(rv.data.decode("utf-8")) + + assert rv.status_code == 422 + assert response == { + "message": {"metadata.yaml": {"type": ["Must be equal to Dashboard."]}} + } diff --git a/tests/databases/commands_tests.py b/tests/databases/commands_tests.py index d1f73ede8a..6b9247dbed 100644 --- a/tests/databases/commands_tests.py +++ b/tests/databases/commands_tests.py @@ -469,7 +469,7 @@ class TestImportDatabasesCommand(SupersetTestCase): command = ImportDatabasesCommand(contents) with pytest.raises(Exception) as excinfo: command.run() - assert str(excinfo.value) == "A wild exception appears!" + assert str(excinfo.value) == "Import database failed for an unknown reason" # verify that the database was not added new_num_databases = db.session.query(Database).count() diff --git a/tests/fixtures/importexport.py b/tests/fixtures/importexport.py index 59e5d6bc27..5f84088e49 100644 --- a/tests/fixtures/importexport.py +++ b/tests/fixtures/importexport.py @@ -103,7 +103,7 @@ dataset_config: Dict[str, Any] = { chart_config: Dict[str, Any] = { "params": { - "color_picker": {"a": 1, "b": 135, "g": 122, "r": 0,}, + "color_picker": {"a": 1, "b": 135, "g": 122, "r": 0}, "datasource": "12__table", "js_columns": ["color"], "js_data_mutator": r"data => data.map(d => ({\n ...d,\n color: colors.hexToRGB(d.extraProps.color)\n}));",