feat: API endpoint to import dashboards (#11751)

* ImportChartsCommand

* feat: add a command to import dashboards

* feat: API endpoint to import dashboards

* Add dispatcher

* Raise specific exception

* Fix test

* Remove print calls

* Add logging when passing

superset/charts/commands/exceptions.py

@@ -23,6 +23,7 @@ from superset.commands.exceptions import (
     CreateFailedError,
     DeleteFailedError,
     ForbiddenError,
+    ImportFailedError,
     UpdateFailedError,
 )
 
@@ -83,3 +84,7 @@ class ChartForbiddenError(ForbiddenError):
 
 class ChartBulkDeleteFailedError(CreateFailedError):
     message = _("Charts could not be deleted.")
+
+
+class ChartImportError(ImportFailedError):
+    message = _("Import chart failed for an unknown reason")

superset/charts/commands/importers/dispatcher.py

@@ -53,8 +53,7 @@ class ImportChartsCommand(BaseCommand):
                 command.run()
                 return
             except IncorrectVersionError:
-                # file is not handled by command, skip
-                pass
+                logger.debug("File not handled by command, skipping")
             except (CommandInvalidError, ValidationError) as exc:
                 # found right version, but file is invalid
                 logger.info("Command failed validation")

superset/charts/commands/importers/v1/__init__.py

@@ -22,6 +22,7 @@ from marshmallow.exceptions import ValidationError
 from sqlalchemy.orm import Session
 
 from superset import db
+from superset.charts.commands.exceptions import ChartImportError
 from superset.charts.commands.importers.v1.utils import import_chart
 from superset.charts.schemas import ImportV1ChartSchema
 from superset.commands.base import BaseCommand
@@ -105,9 +106,9 @@ class ImportChartsCommand(BaseCommand):
         try:
             self._import_bundle(db.session)
             db.session.commit()
-        except Exception as exc:
+        except Exception:
             db.session.rollback()
-            raise exc
+            raise ChartImportError()
 
     def validate(self) -> None:
         exceptions: List[ValidationError] = []

superset/commands/exceptions.py

@@ -73,6 +73,11 @@ class ForbiddenError(CommandException):
     message = "Action is forbidden"
 
 
+class ImportFailedError(CommandException):
+    status = 500
+    message = "Import failed for an unknown reason"
+
+
 class OwnersNotFoundValidationError(ValidationError):
     status = 422
 

superset/dashboards/api.py

@@ -29,6 +29,7 @@ from werkzeug.wrappers import Response as WerkzeugResponse
 from werkzeug.wsgi import FileWrapper
 
 from superset import is_feature_enabled, thumbnail_cache
+from superset.commands.exceptions import CommandInvalidError
 from superset.constants import RouteMethod
 from superset.dashboards.commands.bulk_delete import BulkDeleteDashboardCommand
 from superset.dashboards.commands.create import CreateDashboardCommand
@@ -38,11 +39,13 @@ from superset.dashboards.commands.exceptions import (
     DashboardCreateFailedError,
     DashboardDeleteFailedError,
     DashboardForbiddenError,
+    DashboardImportError,
     DashboardInvalidError,
     DashboardNotFoundError,
     DashboardUpdateFailedError,
 )
 from superset.dashboards.commands.export import ExportDashboardsCommand
+from superset.dashboards.commands.importers.dispatcher import ImportDashboardsCommand
 from superset.dashboards.commands.update import UpdateDashboardCommand
 from superset.dashboards.dao import DashboardDAO
 from superset.dashboards.filters import (
@@ -79,6 +82,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
     datamodel = SQLAInterface(Dashboard)
     include_route_methods = RouteMethod.REST_MODEL_VIEW_CRUD_SET | {
         RouteMethod.EXPORT,
+        RouteMethod.IMPORT,
         RouteMethod.RELATED,
         "bulk_delete",  # not using RouteMethod since locally defined
         "favorite_status",
@@ -641,3 +645,56 @@ class DashboardRestApi(BaseSupersetModelRestApi):
             for request_id in requested_ids
         ]
         return self.response(200, result=res)
+
+    @expose("/import/", methods=["POST"])
+    @protect()
+    @safe
+    @statsd_metrics
+    def import_(self) -> Response:
+        """Import dashboard(s) with associated charts/datasets/databases
+        ---
+        post:
+          requestBody:
+            content:
+              application/zip:
+                schema:
+                  type: string
+                  format: binary
+          responses:
+            200:
+              description: Dashboard import result
+              content:
+                application/json:
+                  schema:
+                    type: object
+                    properties:
+                      message:
+                        type: string
+            400:
+              $ref: '#/components/responses/400'
+            401:
+              $ref: '#/components/responses/401'
+            422:
+              $ref: '#/components/responses/422'
+            500:
+              $ref: '#/components/responses/500'
+        """
+        upload = request.files.get("file")
+        if not upload:
+            return self.response_400()
+        with ZipFile(upload) as bundle:
+            contents = {
+                file_name: bundle.read(file_name).decode()
+                for file_name in bundle.namelist()
+            }
+
+        command = ImportDashboardsCommand(contents)
+        try:
+            command.run()
+            return self.response(200, message="OK")
+        except CommandInvalidError as exc:
+            logger.warning("Import dashboard failed")
+            return self.response_422(message=exc.normalized_messages())
+        except DashboardImportError as exc:
+            logger.exception("Import dashboard failed")
+            return self.response_500(message=str(exc))

superset/dashboards/commands/exceptions.py

@@ -23,6 +23,7 @@ from superset.commands.exceptions import (
     CreateFailedError,
     DeleteFailedError,
     ForbiddenError,
+    ImportFailedError,
     UpdateFailedError,
 )
 
@@ -62,3 +63,7 @@ class DashboardDeleteFailedError(DeleteFailedError):
 
 class DashboardForbiddenError(ForbiddenError):
     message = _("Changing this Dashboard is forbidden")
+
+
+class DashboardImportError(ImportFailedError):
+    message = _("Import dashboard failed for an unknown reason")

superset/dashboards/commands/importers/dispatcher.py

@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+from typing import Any, Dict
+
+from marshmallow.exceptions import ValidationError
+
+from superset.commands.base import BaseCommand
+from superset.commands.exceptions import CommandInvalidError
+from superset.commands.importers.exceptions import IncorrectVersionError
+from superset.dashboards.commands.importers import v0, v1
+
+logger = logging.getLogger(__name__)
+
+# list of different import formats supported; v0 should be last because
+# the files are not versioned
+command_versions = [
+    v1.ImportDashboardsCommand,
+    v0.ImportDashboardsCommand,
+]
+
+
+class ImportDashboardsCommand(BaseCommand):
+    """
+    Import dashboards.
+
+    This command dispatches the import to different versions of the command
+    until it finds one that matches.
+    """
+
+    # pylint: disable=unused-argument
+    def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
+        self.contents = contents
+
+    def run(self) -> None:
+        # iterate over all commands until we find a version that can
+        # handle the contents
+        for version in command_versions:
+            command = version(self.contents)
+            try:
+                command.run()
+                return
+            except IncorrectVersionError:
+                logger.debug("File not handled by command, skipping")
+            except (CommandInvalidError, ValidationError) as exc:
+                # found right version, but file is invalid
+                logger.info("Command failed validation")
+                raise exc
+            except Exception as exc:
+                # validation succeeded but something went wrong
+                logger.exception("Error running import command")
+                raise exc
+
+        raise CommandInvalidError("Could not find a valid command to import file")
+
+    def validate(self) -> None:
+        pass

superset/dashboards/commands/importers/v1/__init__.py

@@ -32,6 +32,7 @@ from superset.commands.importers.v1.utils import (
     load_yaml,
     METADATA_FILE_NAME,
 )
+from superset.dashboards.commands.exceptions import DashboardImportError
 from superset.dashboards.commands.importers.v1.utils import import_dashboard
 from superset.dashboards.schemas import ImportV1DashboardSchema
 from superset.databases.commands.importers.v1.utils import import_database
@@ -154,9 +155,9 @@ class ImportDashboardsCommand(BaseCommand):
         try:
             self._import_bundle(db.session)
             db.session.commit()
-        except Exception as exc:
+        except Exception:
             db.session.rollback()
-            raise exc
+            raise DashboardImportError()
 
     def validate(self) -> None:
         exceptions: List[ValidationError] = []

superset/databases/api.py

@@ -44,6 +44,7 @@ from superset.databases.commands.exceptions import (
     DatabaseCreateFailedError,
     DatabaseDeleteDatasetsExistFailedError,
     DatabaseDeleteFailedError,
+    DatabaseImportError,
     DatabaseInvalidError,
     DatabaseNotFoundError,
     DatabaseSecurityUnsafeError,
@@ -775,6 +776,6 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
         except CommandInvalidError as exc:
             logger.warning("Import database failed")
             return self.response_422(message=exc.normalized_messages())
-        except Exception as exc:  # pylint: disable=broad-except
+        except DatabaseImportError as exc:
             logger.exception("Import database failed")
             return self.response_500(message=str(exc))

superset/databases/commands/exceptions.py

@@ -22,6 +22,7 @@ from superset.commands.exceptions import (
     CommandInvalidError,
     CreateFailedError,
     DeleteFailedError,
+    ImportFailedError,
     UpdateFailedError,
 )
 from superset.security.analytics_db_safety import DBSecurityException
@@ -114,3 +115,7 @@ class DatabaseDeleteFailedError(DeleteFailedError):
 
 class DatabaseSecurityUnsafeError(DBSecurityException):
     message = _("Stopped an unsafe database connection")
+
+
+class DatabaseImportError(ImportFailedError):
+    message = _("Import database failed for an unknown reason")

superset/databases/commands/importers/dispatcher.py

@@ -51,8 +51,7 @@ class ImportDatabasesCommand(BaseCommand):
                 command.run()
                 return
             except IncorrectVersionError:
-                # file is not handled by this command, skip
-                pass
+                logger.debug("File not handled by command, skipping")
             except (CommandInvalidError, ValidationError) as exc:
                 # found right version, but file is invalid
                 logger.info("Command failed validation")

superset/databases/commands/importers/v1/__init__.py

@@ -29,6 +29,7 @@ from superset.commands.importers.v1.utils import (
     load_yaml,
     METADATA_FILE_NAME,
 )
+from superset.databases.commands.exceptions import DatabaseImportError
 from superset.databases.commands.importers.v1.utils import import_database
 from superset.databases.schemas import ImportV1DatabaseSchema
 from superset.datasets.commands.importers.v1.utils import import_dataset
@@ -75,9 +76,9 @@ class ImportDatabasesCommand(BaseCommand):
         try:
             self._import_bundle(db.session)
             db.session.commit()
-        except Exception as exc:
+        except Exception:
             db.session.rollback()
-            raise exc
+            raise DatabaseImportError()
 
     def validate(self) -> None:
         exceptions: List[ValidationError] = []

superset/datasets/api.py

@@ -40,6 +40,7 @@ from superset.datasets.commands.exceptions import (
     DatasetCreateFailedError,
     DatasetDeleteFailedError,
     DatasetForbiddenError,
+    DatasetImportError,
     DatasetInvalidError,
     DatasetNotFoundError,
     DatasetRefreshFailedError,
@@ -598,7 +599,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
     @safe
     @statsd_metrics
     def import_(self) -> Response:
-        """Import dataset (s) with associated databases
+        """Import dataset(s) with associated databases
         ---
         post:
           requestBody:
@@ -642,6 +643,6 @@ class DatasetRestApi(BaseSupersetModelRestApi):
         except CommandInvalidError as exc:
             logger.warning("Import dataset failed")
             return self.response_422(message=exc.normalized_messages())
-        except Exception as exc:  # pylint: disable=broad-except
+        except DatasetImportError as exc:
             logger.exception("Import dataset failed")
             return self.response_500(message=str(exc))

superset/datasets/commands/exceptions.py

@@ -23,6 +23,7 @@ from superset.commands.exceptions import (
     CreateFailedError,
     DeleteFailedError,
     ForbiddenError,
+    ImportFailedError,
     UpdateFailedError,
 )
 from superset.views.base import get_datasource_exist_error_msg
@@ -170,3 +171,7 @@ class DatasetRefreshFailedError(UpdateFailedError):
 
 class DatasetForbiddenError(ForbiddenError):
     message = _("Changing this dataset is forbidden")
+
+
+class DatasetImportError(ImportFailedError):
+    message = _("Import dataset failed for an unknown reason")

superset/datasets/commands/importers/dispatcher.py

@@ -56,8 +56,7 @@ class ImportDatasetsCommand(BaseCommand):
                 command.run()
                 return
             except IncorrectVersionError:
-                # file is not handled by command, skip
-                pass
+                logger.debug("File not handled by command, skipping")
             except (CommandInvalidError, ValidationError) as exc:
                 # found right version, but file is invalid
                 logger.info("Command failed validation")

superset/datasets/commands/importers/v1/__init__.py

@@ -32,6 +32,7 @@ from superset.commands.importers.v1.utils import (
 from superset.connectors.sqla.models import SqlaTable
 from superset.databases.commands.importers.v1.utils import import_database
 from superset.databases.schemas import ImportV1DatabaseSchema
+from superset.datasets.commands.exceptions import DatasetImportError
 from superset.datasets.commands.importers.v1.utils import import_dataset
 from superset.datasets.schemas import ImportV1DatasetSchema
 
@@ -80,9 +81,9 @@ class ImportDatasetsCommand(BaseCommand):
         try:
             self._import_bundle(db.session)
             db.session.commit()
-        except Exception as exc:
+        except Exception:
             db.session.rollback()
-            raise exc
+            raise DatasetImportError()
 
     def validate(self) -> None:
         exceptions: List[ValidationError] = []

tests/dashboards/api_tests.py

@@ -15,18 +15,19 @@
 # specific language governing permissions and limitations
 # under the License.
 # isort:skip_file
+# pylint: disable=too-many-public-methods, no-self-use, invalid-name, too-many-arguments
 """Unit tests for Superset"""
 import json
 from io import BytesIO
 from typing import List, Optional
 from unittest.mock import patch
-from zipfile import is_zipfile
+from zipfile import is_zipfile, ZipFile
 
 import pytest
 import prison
+import yaml
 from sqlalchemy.sql import func
 
-import tests.test_app
 from freezegun import freeze_time
 from sqlalchemy import and_
 from superset import db, security_manager
@@ -37,6 +38,14 @@ from superset.views.base import generate_download_headers
 
 from tests.base_api_tests import ApiOwnersTestCaseMixin
 from tests.base_tests import SupersetTestCase
+from tests.fixtures.importexport import (
+    chart_config,
+    database_config,
+    dashboard_config,
+    dashboard_metadata_config,
+    dataset_config,
+    dataset_metadata_config,
+)
 
 
 DASHBOARDS_FIXTURE_COUNT = 10
@@ -142,7 +151,7 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin):
             ],
             "position_json": "",
             "published": False,
-            "url": f"/superset/dashboard/slug1/",
+            "url": "/superset/dashboard/slug1/",
             "slug": "slug1",
             "table_names": "",
             "thumbnail_url": dashboard.thumbnail_url,
@@ -162,7 +171,7 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin):
         Dashboard API: Test info
         """
         self.login(username="admin")
-        uri = f"api/v1/dashboard/_info"
+        uri = "api/v1/dashboard/_info"
         rv = self.get_assert_metric(uri, "info")
         self.assertEqual(rv.status_code, 200)
 
@@ -1077,3 +1086,86 @@ class TestDashboardApi(SupersetTestCase, ApiOwnersTestCaseMixin):
 
         db.session.delete(dashboard)
         db.session.commit()
+
+    def test_import_dashboard(self):
+        """
+        Dashboard API: Test import dashboard
+        """
+        self.login(username="admin")
+        uri = "api/v1/dashboard/import/"
+
+        buf = BytesIO()
+        with ZipFile(buf, "w") as bundle:
+            with bundle.open("metadata.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(dashboard_metadata_config).encode())
+            with bundle.open("databases/imported_database.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(database_config).encode())
+            with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(dataset_config).encode())
+            with bundle.open("charts/imported_chart.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(chart_config).encode())
+            with bundle.open("dashboards/imported_dashboard.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(dashboard_config).encode())
+        buf.seek(0)
+
+        form_data = {
+            "file": (buf, "dashboard_export.zip"),
+        }
+        rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
+        response = json.loads(rv.data.decode("utf-8"))
+
+        assert rv.status_code == 200
+        assert response == {"message": "OK"}
+
+        dashboard = (
+            db.session.query(Dashboard).filter_by(uuid=dashboard_config["uuid"]).one()
+        )
+        assert dashboard.dashboard_title == "Test dash"
+
+        assert len(dashboard.slices) == 1
+        chart = dashboard.slices[0]
+        assert str(chart.uuid) == chart_config["uuid"]
+
+        dataset = chart.table
+        assert str(dataset.uuid) == dataset_config["uuid"]
+
+        database = dataset.database
+        assert str(database.uuid) == database_config["uuid"]
+
+        db.session.delete(dashboard)
+        db.session.delete(chart)
+        db.session.delete(dataset)
+        db.session.delete(database)
+        db.session.commit()
+
+    def test_import_dashboard_invalid(self):
+        """
+        Dataset API: Test import invalid dashboard
+        """
+        self.login(username="admin")
+        uri = "api/v1/dashboard/import/"
+
+        buf = BytesIO()
+        with ZipFile(buf, "w") as bundle:
+            with bundle.open("metadata.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(dataset_metadata_config).encode())
+            with bundle.open("databases/imported_database.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(database_config).encode())
+            with bundle.open("datasets/imported_dataset.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(dataset_config).encode())
+            with bundle.open("charts/imported_chart.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(chart_config).encode())
+            with bundle.open("dashboards/imported_dashboard.yaml", "w") as fp:
+                fp.write(yaml.safe_dump(dashboard_config).encode())
+        buf.seek(0)
+
+        form_data = {
+            "file": (buf, "dashboard_export.zip"),
+        }
+        rv = self.client.post(uri, data=form_data, content_type="multipart/form-data")
+        response = json.loads(rv.data.decode("utf-8"))
+
+        assert rv.status_code == 422
+        assert response == {
+            "message": {"metadata.yaml": {"type": ["Must be equal to Dashboard."]}}
+        }

tests/databases/commands_tests.py

@@ -469,7 +469,7 @@ class TestImportDatabasesCommand(SupersetTestCase):
         command = ImportDatabasesCommand(contents)
         with pytest.raises(Exception) as excinfo:
             command.run()
-        assert str(excinfo.value) == "A wild exception appears!"
+        assert str(excinfo.value) == "Import database failed for an unknown reason"
 
         # verify that the database was not added
         new_num_databases = db.session.query(Database).count()

tests/fixtures/importexport.py

@@ -103,7 +103,7 @@ dataset_config: Dict[str, Any] = {
 
 chart_config: Dict[str, Any] = {
     "params": {
-        "color_picker": {"a": 1, "b": 135, "g": 122, "r": 0,},
+        "color_picker": {"a": 1, "b": 135, "g": 122, "r": 0},
         "datasource": "12__table",
         "js_columns": ["color"],
         "js_data_mutator": r"data => data.map(d => ({\n    ...d,\n    color: colors.hexToRGB(d.extraProps.color)\n}));",