mirror of https://github.com/apache/superset.git
feat: import external management columns (#19315)
* feat: import flags * Add tests
This commit is contained in:
parent
3313530f4d
commit
c7f9060a2f
|
@ -1304,6 +1304,8 @@ class ImportV1ChartSchema(Schema):
|
|||
uuid = fields.UUID(required=True)
|
||||
version = fields.String(required=True)
|
||||
dataset_uuid = fields.UUID(required=True)
|
||||
is_managed_externally = fields.Boolean(allow_none=True, default=False)
|
||||
external_url = fields.String(allow_none=True)
|
||||
|
||||
|
||||
CHART_SCHEMAS = (
|
||||
|
|
|
@ -117,6 +117,8 @@ class BaseDatasource(
|
|||
owners: List[User]
|
||||
update_from_object_fields: List[str]
|
||||
|
||||
extra_import_fields = ["is_managed_externally", "external_url"]
|
||||
|
||||
@property
|
||||
def kind(self) -> DatasourceKind:
|
||||
return DatasourceKind.VIRTUAL if self.sql else DatasourceKind.PHYSICAL
|
||||
|
|
|
@ -305,3 +305,5 @@ class ImportV1DashboardSchema(Schema):
|
|||
position = fields.Dict()
|
||||
metadata = fields.Dict()
|
||||
version = fields.String(required=True)
|
||||
is_managed_externally = fields.Boolean(allow_none=True, default=False)
|
||||
external_url = fields.String(allow_none=True)
|
||||
|
|
|
@ -623,6 +623,8 @@ class ImportV1DatabaseSchema(Schema):
|
|||
extra = fields.Nested(ImportV1DatabaseExtraSchema)
|
||||
uuid = fields.UUID(required=True)
|
||||
version = fields.String(required=True)
|
||||
is_managed_externally = fields.Boolean(allow_none=True, default=False)
|
||||
external_url = fields.String(allow_none=True)
|
||||
|
||||
# pylint: disable=no-self-use, unused-argument
|
||||
@validates_schema
|
||||
|
|
|
@ -216,6 +216,8 @@ class ImportV1DatasetSchema(Schema):
|
|||
version = fields.String(required=True)
|
||||
database_uuid = fields.UUID(required=True)
|
||||
data = fields.URL()
|
||||
is_managed_externally = fields.Boolean(allow_none=True, default=False)
|
||||
external_url = fields.String(allow_none=True)
|
||||
|
||||
|
||||
class DatasetSchema(SQLAlchemyAutoSchema):
|
||||
|
|
|
@ -165,7 +165,7 @@ class Database(
|
|||
"allow_file_upload",
|
||||
"extra",
|
||||
]
|
||||
extra_import_fields = ["password"]
|
||||
extra_import_fields = ["password", "is_managed_externally", "external_url"]
|
||||
export_children = ["tables"]
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
|
|
@ -163,6 +163,7 @@ class Dashboard(Model, AuditMixinNullable, ImportExportMixin):
|
|||
"css",
|
||||
"slug",
|
||||
]
|
||||
extra_import_fields = ["is_managed_externally", "external_url"]
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Dashboard<{self.id or self.slug}>"
|
||||
|
|
|
@ -119,6 +119,7 @@ class Slice( # pylint: disable=too-many-public-methods
|
|||
"cache_timeout",
|
||||
]
|
||||
export_parent = "table"
|
||||
extra_import_fields = ["is_managed_externally", "external_url"]
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.slice_name or str(self.id)
|
||||
|
|
|
@ -449,7 +449,7 @@ chart_config: Dict[str, Any] = {
|
|||
"dataset_uuid": "10808100-158b-42c4-842e-f32b99d88dfb",
|
||||
}
|
||||
|
||||
dashboard_config = {
|
||||
dashboard_config: Dict[str, Any] = {
|
||||
"dashboard_title": "Test dash",
|
||||
"description": None,
|
||||
"css": "",
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,69 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# pylint: disable=unused-argument, import-outside-toplevel, unused-import, invalid-name
|
||||
|
||||
import copy
|
||||
|
||||
from sqlalchemy.orm.session import Session
|
||||
|
||||
|
||||
def test_import_chart(app_context: None, session: Session) -> None:
|
||||
"""
|
||||
Test importing a chart.
|
||||
"""
|
||||
from superset.charts.commands.importers.v1.utils import import_chart
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
from superset.models.core import Database
|
||||
from superset.models.slice import Slice
|
||||
from tests.integration_tests.fixtures.importexport import chart_config
|
||||
|
||||
engine = session.get_bind()
|
||||
Slice.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(chart_config)
|
||||
config["datasource_id"] = 1
|
||||
config["datasource_type"] = "table"
|
||||
|
||||
chart = import_chart(session, config)
|
||||
assert chart.slice_name == "Deck Path"
|
||||
assert chart.viz_type == "deck_path"
|
||||
assert chart.is_managed_externally is False
|
||||
assert chart.external_url is None
|
||||
|
||||
|
||||
def test_import_chart_managed_externally(app_context: None, session: Session) -> None:
|
||||
"""
|
||||
Test importing a chart that is managed externally.
|
||||
"""
|
||||
from superset.charts.commands.importers.v1.utils import import_chart
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
from superset.models.core import Database
|
||||
from superset.models.slice import Slice
|
||||
from tests.integration_tests.fixtures.importexport import chart_config
|
||||
|
||||
engine = session.get_bind()
|
||||
Slice.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(chart_config)
|
||||
config["datasource_id"] = 1
|
||||
config["datasource_type"] = "table"
|
||||
config["is_managed_externally"] = True
|
||||
config["external_url"] = "https://example.org/my_chart"
|
||||
|
||||
chart = import_chart(session, config)
|
||||
assert chart.is_managed_externally is True
|
||||
assert chart.external_url == "https://example.org/my_chart"
|
|
@ -0,0 +1,67 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# pylint: disable=unused-argument, import-outside-toplevel, unused-import, invalid-name
|
||||
|
||||
import copy
|
||||
|
||||
from sqlalchemy.orm.session import Session
|
||||
|
||||
|
||||
def test_import_dashboard(app_context: None, session: Session) -> None:
|
||||
"""
|
||||
Test importing a dashboard.
|
||||
"""
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
from superset.dashboards.commands.importers.v1.utils import import_dashboard
|
||||
from superset.models.core import Database
|
||||
from superset.models.slice import Slice
|
||||
from tests.integration_tests.fixtures.importexport import dashboard_config
|
||||
|
||||
engine = session.get_bind()
|
||||
Slice.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(dashboard_config)
|
||||
|
||||
dashboard = import_dashboard(session, config)
|
||||
assert dashboard.dashboard_title == "Test dash"
|
||||
assert dashboard.description is None
|
||||
assert dashboard.is_managed_externally is False
|
||||
assert dashboard.external_url is None
|
||||
|
||||
|
||||
def test_import_dashboard_managed_externally(
|
||||
app_context: None, session: Session
|
||||
) -> None:
|
||||
"""
|
||||
Test importing a dashboard that is managed externally.
|
||||
"""
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
from superset.dashboards.commands.importers.v1.utils import import_dashboard
|
||||
from superset.models.core import Database
|
||||
from superset.models.slice import Slice
|
||||
from tests.integration_tests.fixtures.importexport import dashboard_config
|
||||
|
||||
engine = session.get_bind()
|
||||
Slice.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(dashboard_config)
|
||||
config["is_managed_externally"] = True
|
||||
config["external_url"] = "https://example.org/my_dashboard"
|
||||
|
||||
dashboard = import_dashboard(session, config)
|
||||
assert dashboard.is_managed_externally is True
|
||||
assert dashboard.external_url == "https://example.org/my_dashboard"
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,16 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,70 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# pylint: disable=unused-argument, import-outside-toplevel, invalid-name
|
||||
|
||||
import copy
|
||||
|
||||
from sqlalchemy.orm.session import Session
|
||||
|
||||
|
||||
def test_import_database(app_context: None, session: Session) -> None:
|
||||
"""
|
||||
Test importing a database.
|
||||
"""
|
||||
from superset.databases.commands.importers.v1.utils import import_database
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import database_config
|
||||
|
||||
engine = session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(database_config)
|
||||
database = import_database(session, config)
|
||||
assert database.database_name == "imported_database"
|
||||
assert database.sqlalchemy_uri == "sqlite:///test.db"
|
||||
assert database.cache_timeout is None
|
||||
assert database.expose_in_sqllab is True
|
||||
assert database.allow_run_async is False
|
||||
assert database.allow_ctas is True
|
||||
assert database.allow_cvas is True
|
||||
assert database.allow_file_upload is True
|
||||
assert database.extra == "{}"
|
||||
assert database.uuid == "b8a1ccd3-779d-4ab7-8ad8-9ab119d7fe89"
|
||||
assert database.is_managed_externally is False
|
||||
assert database.external_url is None
|
||||
|
||||
|
||||
def test_import_database_managed_externally(
|
||||
app_context: None, session: Session
|
||||
) -> None:
|
||||
"""
|
||||
Test importing a database that is managed externally.
|
||||
"""
|
||||
from superset.databases.commands.importers.v1.utils import import_database
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import database_config
|
||||
|
||||
engine = session.get_bind()
|
||||
Database.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
config = copy.deepcopy(database_config)
|
||||
config["is_managed_externally"] = True
|
||||
config["external_url"] = "https://example.org/my_database"
|
||||
|
||||
database = import_database(session, config)
|
||||
assert database.is_managed_externally is True
|
||||
assert database.external_url == "https://example.org/my_database"
|
|
@ -16,6 +16,7 @@
|
|||
# under the License.
|
||||
# pylint: disable=import-outside-toplevel, unused-argument, unused-import, invalid-name
|
||||
|
||||
import copy
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any, Dict
|
||||
|
@ -199,6 +200,7 @@ def test_import_column_extra_is_string(app_context: None, session: Session) -> N
|
|||
"database_uuid": database.uuid,
|
||||
}
|
||||
|
||||
# the Marshmallow schema should convert strings to objects
|
||||
schema = ImportV1DatasetSchema()
|
||||
dataset_config = schema.load(yaml_config)
|
||||
dataset_config["database_id"] = database.id
|
||||
|
@ -207,3 +209,31 @@ def test_import_column_extra_is_string(app_context: None, session: Session) -> N
|
|||
assert sqla_table.metrics[0].extra == '{"warning_markdown": null}'
|
||||
assert sqla_table.columns[0].extra == '{"certified_by": "User"}'
|
||||
assert sqla_table.extra == '{"warning_markdown": "*WARNING*"}'
|
||||
|
||||
|
||||
def test_import_dataset_managed_externally(app_context: None, session: Session) -> None:
|
||||
"""
|
||||
Test importing a dataset that is managed externally.
|
||||
"""
|
||||
from superset.connectors.sqla.models import SqlaTable, SqlMetric, TableColumn
|
||||
from superset.datasets.commands.importers.v1.utils import import_dataset
|
||||
from superset.datasets.schemas import ImportV1DatasetSchema
|
||||
from superset.models.core import Database
|
||||
from tests.integration_tests.fixtures.importexport import dataset_config
|
||||
|
||||
engine = session.get_bind()
|
||||
SqlaTable.metadata.create_all(engine) # pylint: disable=no-member
|
||||
|
||||
database = Database(database_name="my_database", sqlalchemy_uri="sqlite://")
|
||||
session.add(database)
|
||||
session.flush()
|
||||
|
||||
dataset_uuid = uuid.uuid4()
|
||||
config = copy.deepcopy(dataset_config)
|
||||
config["is_managed_externally"] = True
|
||||
config["external_url"] = "https://example.org/my_table"
|
||||
config["database_id"] = database.id
|
||||
|
||||
sqla_table = import_dataset(session, config)
|
||||
assert sqla_table.is_managed_externally is True
|
||||
assert sqla_table.external_url == "https://example.org/my_table"
|
||||
|
|
Loading…
Reference in New Issue