feat: import external management columns (#19315)

* feat: import flags

* Add tests

superset/charts/schemas.py

@@ -1304,6 +1304,8 @@ class ImportV1ChartSchema(Schema):
     uuid = fields.UUID(required=True)
     version = fields.String(required=True)
     dataset_uuid = fields.UUID(required=True)
+    is_managed_externally = fields.Boolean(allow_none=True, default=False)
+    external_url = fields.String(allow_none=True)
 
 
 CHART_SCHEMAS = (

superset/connectors/base/models.py

@@ -117,6 +117,8 @@ class BaseDatasource(
     owners: List[User]
     update_from_object_fields: List[str]
 
+    extra_import_fields = ["is_managed_externally", "external_url"]
+
     @property
     def kind(self) -> DatasourceKind:
         return DatasourceKind.VIRTUAL if self.sql else DatasourceKind.PHYSICAL

superset/dashboards/schemas.py

@@ -305,3 +305,5 @@ class ImportV1DashboardSchema(Schema):
     position = fields.Dict()
     metadata = fields.Dict()
     version = fields.String(required=True)
+    is_managed_externally = fields.Boolean(allow_none=True, default=False)
+    external_url = fields.String(allow_none=True)

superset/databases/schemas.py

@@ -623,6 +623,8 @@ class ImportV1DatabaseSchema(Schema):
     extra = fields.Nested(ImportV1DatabaseExtraSchema)
     uuid = fields.UUID(required=True)
     version = fields.String(required=True)
+    is_managed_externally = fields.Boolean(allow_none=True, default=False)
+    external_url = fields.String(allow_none=True)
 
     # pylint: disable=no-self-use, unused-argument
     @validates_schema

superset/datasets/schemas.py

@@ -216,6 +216,8 @@ class ImportV1DatasetSchema(Schema):
     version = fields.String(required=True)
     database_uuid = fields.UUID(required=True)
     data = fields.URL()
+    is_managed_externally = fields.Boolean(allow_none=True, default=False)
+    external_url = fields.String(allow_none=True)
 
 
 class DatasetSchema(SQLAlchemyAutoSchema):

superset/models/core.py

@@ -165,7 +165,7 @@ class Database(
         "allow_file_upload",
         "extra",
     ]
-    extra_import_fields = ["password"]
+    extra_import_fields = ["password", "is_managed_externally", "external_url"]
     export_children = ["tables"]
 
     def __repr__(self) -> str:

superset/models/dashboard.py

@@ -163,6 +163,7 @@ class Dashboard(Model, AuditMixinNullable, ImportExportMixin):
         "css",
         "slug",
     ]
+    extra_import_fields = ["is_managed_externally", "external_url"]
 
     def __repr__(self) -> str:
         return f"Dashboard<{self.id or self.slug}>"

superset/models/slice.py

@@ -119,6 +119,7 @@ class Slice(  # pylint: disable=too-many-public-methods
         "cache_timeout",
     ]
     export_parent = "table"
+    extra_import_fields = ["is_managed_externally", "external_url"]
 
     def __repr__(self) -> str:
         return self.slice_name or str(self.id)

tests/integration_tests/fixtures/importexport.py

@@ -449,7 +449,7 @@ chart_config: Dict[str, Any] = {
     "dataset_uuid": "10808100-158b-42c4-842e-f32b99d88dfb",
 }
 
-dashboard_config = {
+dashboard_config: Dict[str, Any] = {
     "dashboard_title": "Test dash",
     "description": None,
     "css": "",

tests/unit_tests/charts/commands/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/charts/commands/importers/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/charts/commands/importers/v1/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/charts/commands/importers/v1/import_test.py

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=unused-argument, import-outside-toplevel, unused-import, invalid-name
+
+import copy
+
+from sqlalchemy.orm.session import Session
+
+
+def test_import_chart(app_context: None, session: Session) -> None:
+    """
+    Test importing a chart.
+    """
+    from superset.charts.commands.importers.v1.utils import import_chart
+    from superset.connectors.sqla.models import SqlaTable
+    from superset.models.core import Database
+    from superset.models.slice import Slice
+    from tests.integration_tests.fixtures.importexport import chart_config
+
+    engine = session.get_bind()
+    Slice.metadata.create_all(engine)  # pylint: disable=no-member
+
+    config = copy.deepcopy(chart_config)
+    config["datasource_id"] = 1
+    config["datasource_type"] = "table"
+
+    chart = import_chart(session, config)
+    assert chart.slice_name == "Deck Path"
+    assert chart.viz_type == "deck_path"
+    assert chart.is_managed_externally is False
+    assert chart.external_url is None
+
+
+def test_import_chart_managed_externally(app_context: None, session: Session) -> None:
+    """
+    Test importing a chart that is managed externally.
+    """
+    from superset.charts.commands.importers.v1.utils import import_chart
+    from superset.connectors.sqla.models import SqlaTable
+    from superset.models.core import Database
+    from superset.models.slice import Slice
+    from tests.integration_tests.fixtures.importexport import chart_config
+
+    engine = session.get_bind()
+    Slice.metadata.create_all(engine)  # pylint: disable=no-member
+
+    config = copy.deepcopy(chart_config)
+    config["datasource_id"] = 1
+    config["datasource_type"] = "table"
+    config["is_managed_externally"] = True
+    config["external_url"] = "https://example.org/my_chart"
+
+    chart = import_chart(session, config)
+    assert chart.is_managed_externally is True
+    assert chart.external_url == "https://example.org/my_chart"

tests/unit_tests/dashboards/commands/importers/v1/import_test.py

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=unused-argument, import-outside-toplevel, unused-import, invalid-name
+
+import copy
+
+from sqlalchemy.orm.session import Session
+
+
+def test_import_dashboard(app_context: None, session: Session) -> None:
+    """
+    Test importing a dashboard.
+    """
+    from superset.connectors.sqla.models import SqlaTable
+    from superset.dashboards.commands.importers.v1.utils import import_dashboard
+    from superset.models.core import Database
+    from superset.models.slice import Slice
+    from tests.integration_tests.fixtures.importexport import dashboard_config
+
+    engine = session.get_bind()
+    Slice.metadata.create_all(engine)  # pylint: disable=no-member
+
+    config = copy.deepcopy(dashboard_config)
+
+    dashboard = import_dashboard(session, config)
+    assert dashboard.dashboard_title == "Test dash"
+    assert dashboard.description is None
+    assert dashboard.is_managed_externally is False
+    assert dashboard.external_url is None
+
+
+def test_import_dashboard_managed_externally(
+    app_context: None, session: Session
+) -> None:
+    """
+    Test importing a dashboard that is managed externally.
+    """
+    from superset.connectors.sqla.models import SqlaTable
+    from superset.dashboards.commands.importers.v1.utils import import_dashboard
+    from superset.models.core import Database
+    from superset.models.slice import Slice
+    from tests.integration_tests.fixtures.importexport import dashboard_config
+
+    engine = session.get_bind()
+    Slice.metadata.create_all(engine)  # pylint: disable=no-member
+
+    config = copy.deepcopy(dashboard_config)
+    config["is_managed_externally"] = True
+    config["external_url"] = "https://example.org/my_dashboard"
+
+    dashboard = import_dashboard(session, config)
+    assert dashboard.is_managed_externally is True
+    assert dashboard.external_url == "https://example.org/my_dashboard"

tests/unit_tests/databases/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/databases/commands/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/databases/commands/importers/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/databases/commands/importers/v1/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/unit_tests/databases/commands/importers/v1/import_test.py

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=unused-argument, import-outside-toplevel, invalid-name
+
+import copy
+
+from sqlalchemy.orm.session import Session
+
+
+def test_import_database(app_context: None, session: Session) -> None:
+    """
+    Test importing a database.
+    """
+    from superset.databases.commands.importers.v1.utils import import_database
+    from superset.models.core import Database
+    from tests.integration_tests.fixtures.importexport import database_config
+
+    engine = session.get_bind()
+    Database.metadata.create_all(engine)  # pylint: disable=no-member
+
+    config = copy.deepcopy(database_config)
+    database = import_database(session, config)
+    assert database.database_name == "imported_database"
+    assert database.sqlalchemy_uri == "sqlite:///test.db"
+    assert database.cache_timeout is None
+    assert database.expose_in_sqllab is True
+    assert database.allow_run_async is False
+    assert database.allow_ctas is True
+    assert database.allow_cvas is True
+    assert database.allow_file_upload is True
+    assert database.extra == "{}"
+    assert database.uuid == "b8a1ccd3-779d-4ab7-8ad8-9ab119d7fe89"
+    assert database.is_managed_externally is False
+    assert database.external_url is None
+
+
+def test_import_database_managed_externally(
+    app_context: None, session: Session
+) -> None:
+    """
+    Test importing a database that is managed externally.
+    """
+    from superset.databases.commands.importers.v1.utils import import_database
+    from superset.models.core import Database
+    from tests.integration_tests.fixtures.importexport import database_config
+
+    engine = session.get_bind()
+    Database.metadata.create_all(engine)  # pylint: disable=no-member
+
+    config = copy.deepcopy(database_config)
+    config["is_managed_externally"] = True
+    config["external_url"] = "https://example.org/my_database"
+
+    database = import_database(session, config)
+    assert database.is_managed_externally is True
+    assert database.external_url == "https://example.org/my_database"

tests/unit_tests/datasets/commands/importers/v1/import_test.py

@@ -16,6 +16,7 @@
 # under the License.
 # pylint: disable=import-outside-toplevel, unused-argument, unused-import, invalid-name
 
+import copy
 import json
 import uuid
 from typing import Any, Dict
@@ -199,6 +200,7 @@ def test_import_column_extra_is_string(app_context: None, session: Session) -> N
         "database_uuid": database.uuid,
     }
 
+    # the Marshmallow schema should convert strings to objects
     schema = ImportV1DatasetSchema()
     dataset_config = schema.load(yaml_config)
     dataset_config["database_id"] = database.id
@@ -207,3 +209,31 @@ def test_import_column_extra_is_string(app_context: None, session: Session) -> N
     assert sqla_table.metrics[0].extra == '{"warning_markdown": null}'
     assert sqla_table.columns[0].extra == '{"certified_by": "User"}'
     assert sqla_table.extra == '{"warning_markdown": "*WARNING*"}'
+
+
+def test_import_dataset_managed_externally(app_context: None, session: Session) -> None:
+    """
+    Test importing a dataset that is managed externally.
+    """
+    from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn
+    from superset.datasets.commands.importers.v1.utils import import_dataset
+    from superset.datasets.schemas import ImportV1DatasetSchema
+    from superset.models.core import Database
+    from tests.integration_tests.fixtures.importexport import dataset_config
+
+    engine = session.get_bind()
+    SqlaTable.metadata.create_all(engine)  # pylint: disable=no-member
+
+    database = Database(database_name="my_database", sqlalchemy_uri="sqlite://")
+    session.add(database)
+    session.flush()
+
+    dataset_uuid = uuid.uuid4()
+    config = copy.deepcopy(dataset_config)
+    config["is_managed_externally"] = True
+    config["external_url"] = "https://example.org/my_table"
+    config["database_id"] = database.id
+
+    sqla_table = import_dataset(session, config)
+    assert sqla_table.is_managed_externally is True
+    assert sqla_table.external_url == "https://example.org/my_table"