mirror of https://github.com/apache/superset.git
feat: new import commands for dataset and databases (#11670)
* feat: commands for importing databases and datasets * Refactor code
This commit is contained in:
parent
871a98abe2
commit
7bc353f8a8
|
@ -304,6 +304,9 @@ def import_datasources(path: str, sync: str, recursive: bool) -> None:
|
||||||
from superset.datasets.commands.importers.v0 import ImportDatasetsCommand
|
from superset.datasets.commands.importers.v0 import ImportDatasetsCommand
|
||||||
|
|
||||||
sync_array = sync.split(",")
|
sync_array = sync.split(",")
|
||||||
|
sync_columns = "columns" in sync_array
|
||||||
|
sync_metrics = "metrics" in sync_array
|
||||||
|
|
||||||
path_object = Path(path)
|
path_object = Path(path)
|
||||||
files: List[Path] = []
|
files: List[Path] = []
|
||||||
if path_object.is_file():
|
if path_object.is_file():
|
||||||
|
@ -316,7 +319,7 @@ def import_datasources(path: str, sync: str, recursive: bool) -> None:
|
||||||
files.extend(path_object.rglob("*.yml"))
|
files.extend(path_object.rglob("*.yml"))
|
||||||
contents = {path.name: open(path).read() for path in files}
|
contents = {path.name: open(path).read() for path in files}
|
||||||
try:
|
try:
|
||||||
ImportDatasetsCommand(contents, sync_array).run()
|
ImportDatasetsCommand(contents, sync_columns, sync_metrics).run()
|
||||||
except Exception: # pylint: disable=broad-except
|
except Exception: # pylint: disable=broad-except
|
||||||
logger.exception("Error when importing dataset")
|
logger.exception("Error when importing dataset")
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from superset.commands.exceptions import CommandException
|
||||||
|
|
||||||
|
|
||||||
|
class IncorrectVersionError(CommandException):
|
||||||
|
status = 422
|
||||||
|
message = "Import has incorrect version"
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
|
@ -0,0 +1,67 @@
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from marshmallow import fields, Schema, validate
|
||||||
|
from marshmallow.exceptions import ValidationError
|
||||||
|
|
||||||
|
from superset.commands.importers.exceptions import IncorrectVersionError
|
||||||
|
|
||||||
|
METADATA_FILE_NAME = "metadata.yaml"
|
||||||
|
IMPORT_VERSION = "1.0.0"
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class MetadataSchema(Schema):
|
||||||
|
version = fields.String(required=True, validate=validate.Equal(IMPORT_VERSION))
|
||||||
|
type = fields.String(required=True)
|
||||||
|
timestamp = fields.DateTime()
|
||||||
|
|
||||||
|
|
||||||
|
def load_yaml(file_name: str, content: str) -> Dict[str, Any]:
|
||||||
|
"""Try to load a YAML file"""
|
||||||
|
try:
|
||||||
|
return yaml.safe_load(content)
|
||||||
|
except yaml.parser.ParserError:
|
||||||
|
logger.exception("Invalid YAML in %s", METADATA_FILE_NAME)
|
||||||
|
raise ValidationError({file_name: "Not a valid YAML file"})
|
||||||
|
|
||||||
|
|
||||||
|
def load_metadata(contents: Dict[str, str]) -> Dict[str, str]:
|
||||||
|
"""Apply validation and load a metadata file"""
|
||||||
|
if METADATA_FILE_NAME not in contents:
|
||||||
|
# if the contents ahve no METADATA_FILE_NAME this is probably
|
||||||
|
# a original export without versioning that should not be
|
||||||
|
# handled by this command
|
||||||
|
raise IncorrectVersionError(f"Missing {METADATA_FILE_NAME}")
|
||||||
|
|
||||||
|
metadata = load_yaml(METADATA_FILE_NAME, contents[METADATA_FILE_NAME])
|
||||||
|
try:
|
||||||
|
MetadataSchema().load(metadata)
|
||||||
|
except ValidationError as exc:
|
||||||
|
# if the version doesn't match raise an exception so that the
|
||||||
|
# dispatcher can try a different command version
|
||||||
|
if "version" in exc.messages:
|
||||||
|
raise IncorrectVersionError(exc.messages["version"][0])
|
||||||
|
|
||||||
|
# otherwise we raise the validation error
|
||||||
|
exc.messages = {METADATA_FILE_NAME: exc.messages}
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
return metadata
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
|
@ -0,0 +1,116 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from marshmallow import Schema, validate
|
||||||
|
from marshmallow.exceptions import ValidationError
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from superset import db
|
||||||
|
from superset.commands.base import BaseCommand
|
||||||
|
from superset.commands.exceptions import CommandInvalidError
|
||||||
|
from superset.commands.importers.v1.utils import (
|
||||||
|
load_metadata,
|
||||||
|
load_yaml,
|
||||||
|
METADATA_FILE_NAME,
|
||||||
|
)
|
||||||
|
from superset.databases.commands.importers.v1.utils import import_database
|
||||||
|
from superset.databases.schemas import ImportV1DatabaseSchema
|
||||||
|
from superset.datasets.commands.importers.v1.utils import import_dataset
|
||||||
|
from superset.datasets.schemas import ImportV1DatasetSchema
|
||||||
|
from superset.models.core import Database
|
||||||
|
|
||||||
|
schemas: Dict[str, Schema] = {
|
||||||
|
"databases/": ImportV1DatabaseSchema(),
|
||||||
|
"datasets/": ImportV1DatasetSchema(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ImportDatabasesCommand(BaseCommand):
|
||||||
|
|
||||||
|
"""Import databases"""
|
||||||
|
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
|
||||||
|
self.contents = contents
|
||||||
|
self._configs: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
def _import_bundle(self, session: Session) -> None:
|
||||||
|
# first import databases
|
||||||
|
database_ids: Dict[str, int] = {}
|
||||||
|
for file_name, config in self._configs.items():
|
||||||
|
if file_name.startswith("databases/"):
|
||||||
|
database = import_database(session, config, overwrite=True)
|
||||||
|
database_ids[str(database.uuid)] = database.id
|
||||||
|
|
||||||
|
# import related datasets
|
||||||
|
for file_name, config in self._configs.items():
|
||||||
|
if (
|
||||||
|
file_name.startswith("datasets/")
|
||||||
|
and config["database_uuid"] in database_ids
|
||||||
|
):
|
||||||
|
config["database_id"] = database_ids[config["database_uuid"]]
|
||||||
|
# overwrite=False prevents deleting any non-imported columns/metrics
|
||||||
|
import_dataset(session, config, overwrite=False)
|
||||||
|
|
||||||
|
def run(self) -> None:
|
||||||
|
self.validate()
|
||||||
|
|
||||||
|
# rollback to prevent partial imports
|
||||||
|
try:
|
||||||
|
self._import_bundle(db.session)
|
||||||
|
db.session.commit()
|
||||||
|
except Exception as exc:
|
||||||
|
db.session.rollback()
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
def validate(self) -> None:
|
||||||
|
exceptions: List[ValidationError] = []
|
||||||
|
|
||||||
|
# verify that the metadata file is present and valid
|
||||||
|
try:
|
||||||
|
metadata = load_metadata(self.contents)
|
||||||
|
except ValidationError as exc:
|
||||||
|
exceptions.append(exc)
|
||||||
|
metadata = None
|
||||||
|
|
||||||
|
for file_name, content in self.contents.items():
|
||||||
|
prefix = file_name.split("/")[0]
|
||||||
|
schema = schemas.get(f"{prefix}/")
|
||||||
|
if schema:
|
||||||
|
try:
|
||||||
|
config = load_yaml(file_name, content)
|
||||||
|
schema.load(config)
|
||||||
|
self._configs[file_name] = config
|
||||||
|
except ValidationError as exc:
|
||||||
|
exc.messages = {file_name: exc.messages}
|
||||||
|
exceptions.append(exc)
|
||||||
|
|
||||||
|
# validate that the type declared in METADATA_FILE_NAME is correct
|
||||||
|
if metadata:
|
||||||
|
type_validator = validate.Equal(Database.__name__)
|
||||||
|
try:
|
||||||
|
type_validator(metadata["type"])
|
||||||
|
except ValidationError as exc:
|
||||||
|
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
|
||||||
|
exceptions.append(exc)
|
||||||
|
|
||||||
|
if exceptions:
|
||||||
|
exception = CommandInvalidError("Error importing database")
|
||||||
|
exception.add_list(exceptions)
|
||||||
|
raise exception
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from superset.models.core import Database
|
||||||
|
|
||||||
|
|
||||||
|
def import_database(
|
||||||
|
session: Session, config: Dict[str, Any], overwrite: bool = False
|
||||||
|
) -> Database:
|
||||||
|
existing = session.query(Database).filter_by(uuid=config["uuid"]).first()
|
||||||
|
if existing:
|
||||||
|
if not overwrite:
|
||||||
|
return existing
|
||||||
|
config["id"] = existing.id
|
||||||
|
|
||||||
|
# TODO (betodealmeida): move this logic to import_from_dict
|
||||||
|
config["extra"] = json.dumps(config["extra"])
|
||||||
|
|
||||||
|
database = Database.import_from_dict(session, config, recursive=False)
|
||||||
|
if database.id is None:
|
||||||
|
session.flush()
|
||||||
|
|
||||||
|
return database
|
|
@ -408,3 +408,24 @@ class DatabaseRelatedDashboards(Schema):
|
||||||
class DatabaseRelatedObjectsResponse(Schema):
|
class DatabaseRelatedObjectsResponse(Schema):
|
||||||
charts = fields.Nested(DatabaseRelatedCharts)
|
charts = fields.Nested(DatabaseRelatedCharts)
|
||||||
dashboards = fields.Nested(DatabaseRelatedDashboards)
|
dashboards = fields.Nested(DatabaseRelatedDashboards)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportV1DatabaseExtraSchema(Schema):
|
||||||
|
metadata_params = fields.Dict(keys=fields.Str(), values=fields.Raw())
|
||||||
|
engine_params = fields.Dict(keys=fields.Str(), values=fields.Raw())
|
||||||
|
metadata_cache_timeout = fields.Dict(keys=fields.Str(), values=fields.Integer())
|
||||||
|
schemas_allowed_for_csv_upload = fields.List(fields.String)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportV1DatabaseSchema(Schema):
|
||||||
|
database_name = fields.String(required=True)
|
||||||
|
sqlalchemy_uri = fields.String(required=True)
|
||||||
|
cache_timeout = fields.Integer(allow_none=True)
|
||||||
|
expose_in_sqllab = fields.Boolean()
|
||||||
|
allow_run_async = fields.Boolean()
|
||||||
|
allow_ctas = fields.Boolean()
|
||||||
|
allow_cvas = fields.Boolean()
|
||||||
|
allow_csv_upload = fields.Boolean()
|
||||||
|
extra = fields.Nested(ImportV1DatabaseExtraSchema)
|
||||||
|
uuid = fields.UUID(required=True)
|
||||||
|
version = fields.String(required=True)
|
||||||
|
|
|
@ -282,9 +282,19 @@ class ImportDatasetsCommand(BaseCommand):
|
||||||
in Superset.
|
in Superset.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, contents: Dict[str, str], sync: Optional[List[str]] = None):
|
def __init__(
|
||||||
|
self,
|
||||||
|
contents: Dict[str, str],
|
||||||
|
sync_columns: bool = False,
|
||||||
|
sync_metrics: bool = False,
|
||||||
|
):
|
||||||
self.contents = contents
|
self.contents = contents
|
||||||
self.sync = sync
|
|
||||||
|
self.sync = []
|
||||||
|
if sync_columns:
|
||||||
|
self.sync.append("columns")
|
||||||
|
if sync_metrics:
|
||||||
|
self.sync.append("metrics")
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
self.validate()
|
self.validate()
|
||||||
|
|
|
@ -0,0 +1,121 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Set
|
||||||
|
|
||||||
|
from marshmallow import Schema, validate
|
||||||
|
from marshmallow.exceptions import ValidationError
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from superset import db
|
||||||
|
from superset.commands.base import BaseCommand
|
||||||
|
from superset.commands.exceptions import CommandInvalidError
|
||||||
|
from superset.commands.importers.v1.utils import (
|
||||||
|
load_metadata,
|
||||||
|
load_yaml,
|
||||||
|
METADATA_FILE_NAME,
|
||||||
|
)
|
||||||
|
from superset.connectors.sqla.models import SqlaTable
|
||||||
|
from superset.databases.commands.importers.v1.utils import import_database
|
||||||
|
from superset.databases.schemas import ImportV1DatabaseSchema
|
||||||
|
from superset.datasets.commands.importers.v1.utils import import_dataset
|
||||||
|
from superset.datasets.schemas import ImportV1DatasetSchema
|
||||||
|
|
||||||
|
schemas: Dict[str, Schema] = {
|
||||||
|
"databases/": ImportV1DatabaseSchema(),
|
||||||
|
"datasets/": ImportV1DatasetSchema(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ImportDatasetsCommand(BaseCommand):
|
||||||
|
|
||||||
|
"""Import datasets"""
|
||||||
|
|
||||||
|
# pylint: disable=unused-argument
|
||||||
|
def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
|
||||||
|
self.contents = contents
|
||||||
|
self._configs: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
def _import_bundle(self, session: Session) -> None:
|
||||||
|
# discover databases associated with datasets
|
||||||
|
database_uuids: Set[str] = set()
|
||||||
|
for file_name, config in self._configs.items():
|
||||||
|
if file_name.startswith("datasets/"):
|
||||||
|
database_uuids.add(config["database_uuid"])
|
||||||
|
|
||||||
|
# import related databases
|
||||||
|
database_ids: Dict[str, int] = {}
|
||||||
|
for file_name, config in self._configs.items():
|
||||||
|
if file_name.startswith("databases/") and config["uuid"] in database_uuids:
|
||||||
|
database = import_database(session, config, overwrite=False)
|
||||||
|
database_ids[str(database.uuid)] = database.id
|
||||||
|
|
||||||
|
# import datasets with the correct parent ref
|
||||||
|
for file_name, config in self._configs.items():
|
||||||
|
if (
|
||||||
|
file_name.startswith("datasets/")
|
||||||
|
and config["database_uuid"] in database_ids
|
||||||
|
):
|
||||||
|
config["database_id"] = database_ids[config["database_uuid"]]
|
||||||
|
import_dataset(session, config, overwrite=True)
|
||||||
|
|
||||||
|
def run(self) -> None:
|
||||||
|
self.validate()
|
||||||
|
|
||||||
|
# rollback to prevent partial imports
|
||||||
|
try:
|
||||||
|
self._import_bundle(db.session)
|
||||||
|
db.session.commit()
|
||||||
|
except Exception as exc:
|
||||||
|
db.session.rollback()
|
||||||
|
raise exc
|
||||||
|
|
||||||
|
def validate(self) -> None:
|
||||||
|
exceptions: List[ValidationError] = []
|
||||||
|
|
||||||
|
# verify that the metadata file is present and valid
|
||||||
|
try:
|
||||||
|
metadata = load_metadata(self.contents)
|
||||||
|
except ValidationError as exc:
|
||||||
|
exceptions.append(exc)
|
||||||
|
metadata = None
|
||||||
|
|
||||||
|
for file_name, content in self.contents.items():
|
||||||
|
prefix = file_name.split("/")[0]
|
||||||
|
schema = schemas.get(f"{prefix}/")
|
||||||
|
if schema:
|
||||||
|
try:
|
||||||
|
config = load_yaml(file_name, content)
|
||||||
|
schema.load(config)
|
||||||
|
self._configs[file_name] = config
|
||||||
|
except ValidationError as exc:
|
||||||
|
exc.messages = {file_name: exc.messages}
|
||||||
|
exceptions.append(exc)
|
||||||
|
|
||||||
|
# validate that the type declared in METADATA_FILE_NAME is correct
|
||||||
|
if metadata:
|
||||||
|
type_validator = validate.Equal(SqlaTable.__name__)
|
||||||
|
try:
|
||||||
|
type_validator(metadata["type"])
|
||||||
|
except ValidationError as exc:
|
||||||
|
exc.messages = {METADATA_FILE_NAME: {"type": exc.messages}}
|
||||||
|
exceptions.append(exc)
|
||||||
|
|
||||||
|
if exceptions:
|
||||||
|
exception = CommandInvalidError("Error importing dataset")
|
||||||
|
exception.add_list(exceptions)
|
||||||
|
raise exception
|
|
@ -0,0 +1,42 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from superset.connectors.sqla.models import SqlaTable
|
||||||
|
|
||||||
|
|
||||||
|
def import_dataset(
|
||||||
|
session: Session, config: Dict[str, Any], overwrite: bool = False
|
||||||
|
) -> SqlaTable:
|
||||||
|
existing = session.query(SqlaTable).filter_by(uuid=config["uuid"]).first()
|
||||||
|
if existing:
|
||||||
|
if not overwrite:
|
||||||
|
return existing
|
||||||
|
config["id"] = existing.id
|
||||||
|
|
||||||
|
# should we delete columns and metrics not present in the current import?
|
||||||
|
sync = ["columns", "metrics"] if overwrite else []
|
||||||
|
|
||||||
|
# import recursively to include columns and metrics
|
||||||
|
dataset = SqlaTable.import_from_dict(session, config, recursive=True, sync=sync)
|
||||||
|
if dataset.id is None:
|
||||||
|
session.flush()
|
||||||
|
|
||||||
|
return dataset
|
|
@ -122,3 +122,48 @@ class DatasetRelatedDashboards(Schema):
|
||||||
class DatasetRelatedObjectsResponse(Schema):
|
class DatasetRelatedObjectsResponse(Schema):
|
||||||
charts = fields.Nested(DatasetRelatedCharts)
|
charts = fields.Nested(DatasetRelatedCharts)
|
||||||
dashboards = fields.Nested(DatasetRelatedDashboards)
|
dashboards = fields.Nested(DatasetRelatedDashboards)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportV1ColumnSchema(Schema):
|
||||||
|
column_name = fields.String(required=True)
|
||||||
|
verbose_name = fields.String()
|
||||||
|
is_dttm = fields.Boolean()
|
||||||
|
is_active = fields.Boolean(allow_none=True)
|
||||||
|
type = fields.String(required=True)
|
||||||
|
groupby = fields.Boolean()
|
||||||
|
filterable = fields.Boolean()
|
||||||
|
expression = fields.String()
|
||||||
|
description = fields.String(allow_none=True)
|
||||||
|
python_date_format = fields.String(allow_none=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportV1MetricSchema(Schema):
|
||||||
|
metric_name = fields.String(required=True)
|
||||||
|
verbose_name = fields.String()
|
||||||
|
metric_type = fields.String(allow_none=True)
|
||||||
|
expression = fields.String(required=True)
|
||||||
|
description = fields.String(allow_none=True)
|
||||||
|
d3format = fields.String(allow_none=True)
|
||||||
|
extra = fields.String(allow_none=True)
|
||||||
|
warning_text = fields.String(allow_none=True)
|
||||||
|
|
||||||
|
|
||||||
|
class ImportV1DatasetSchema(Schema):
|
||||||
|
table_name = fields.String(required=True)
|
||||||
|
main_dttm_col = fields.String(allow_none=True)
|
||||||
|
description = fields.String()
|
||||||
|
default_endpoint = fields.String()
|
||||||
|
offset = fields.Integer()
|
||||||
|
cache_timeout = fields.Integer()
|
||||||
|
schema = fields.String()
|
||||||
|
sql = fields.String()
|
||||||
|
params = fields.String(allow_none=True)
|
||||||
|
template_params = fields.String(allow_none=True)
|
||||||
|
filter_select_enabled = fields.Boolean()
|
||||||
|
fetch_values_predicate = fields.String(allow_none=True)
|
||||||
|
extra = fields.String(allow_none=True)
|
||||||
|
uuid = fields.UUID(required=True)
|
||||||
|
columns = fields.List(fields.Nested(ImportV1ColumnSchema))
|
||||||
|
metrics = fields.List(fields.Nested(ImportV1MetricSchema))
|
||||||
|
version = fields.String(required=True)
|
||||||
|
database_uuid = fields.UUID(required=True)
|
||||||
|
|
|
@ -163,7 +163,7 @@ class ImportExportMixin:
|
||||||
if sync is None:
|
if sync is None:
|
||||||
sync = []
|
sync = []
|
||||||
parent_refs = cls.parent_foreign_key_mappings()
|
parent_refs = cls.parent_foreign_key_mappings()
|
||||||
export_fields = set(cls.export_fields) | set(parent_refs.keys())
|
export_fields = set(cls.export_fields) | set(parent_refs.keys()) | {"uuid"}
|
||||||
new_children = {c: dict_rep[c] for c in cls.export_children if c in dict_rep}
|
new_children = {c: dict_rep[c] for c in cls.export_children if c in dict_rep}
|
||||||
unique_constrains = cls._unique_constrains()
|
unique_constrains = cls._unique_constrains()
|
||||||
|
|
||||||
|
|
|
@ -14,16 +14,29 @@
|
||||||
# KIND, either express or implied. See the License for the
|
# KIND, either express or implied. See the License for the
|
||||||
# specific language governing permissions and limitations
|
# specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
# pylint: disable=no-self-use, invalid-name
|
||||||
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from superset import security_manager
|
from superset import db, security_manager
|
||||||
|
from superset.commands.exceptions import CommandInvalidError
|
||||||
|
from superset.commands.importers.exceptions import IncorrectVersionError
|
||||||
|
from superset.connectors.sqla.models import SqlaTable
|
||||||
from superset.databases.commands.exceptions import DatabaseNotFoundError
|
from superset.databases.commands.exceptions import DatabaseNotFoundError
|
||||||
from superset.databases.commands.export import ExportDatabasesCommand
|
from superset.databases.commands.export import ExportDatabasesCommand
|
||||||
|
from superset.databases.commands.importers.v1 import ImportDatabasesCommand
|
||||||
|
from superset.models.core import Database
|
||||||
from superset.utils.core import backend, get_example_database
|
from superset.utils.core import backend, get_example_database
|
||||||
from tests.base_tests import SupersetTestCase
|
from tests.base_tests import SupersetTestCase
|
||||||
|
from tests.fixtures.importexport import (
|
||||||
|
database_config,
|
||||||
|
database_metadata_config,
|
||||||
|
dataset_config,
|
||||||
|
dataset_metadata_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestExportDatabasesCommand(SupersetTestCase):
|
class TestExportDatabasesCommand(SupersetTestCase):
|
||||||
|
@ -265,3 +278,197 @@ class TestExportDatabasesCommand(SupersetTestCase):
|
||||||
"uuid",
|
"uuid",
|
||||||
"version",
|
"version",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def test_import_v1_database(self):
|
||||||
|
"""Test that a database can be imported"""
|
||||||
|
contents = {
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
database = (
|
||||||
|
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert database.allow_csv_upload
|
||||||
|
assert database.allow_ctas
|
||||||
|
assert database.allow_cvas
|
||||||
|
assert not database.allow_run_async
|
||||||
|
assert database.cache_timeout is None
|
||||||
|
assert database.database_name == "imported_database"
|
||||||
|
assert database.expose_in_sqllab
|
||||||
|
assert database.extra == "{}"
|
||||||
|
assert database.sqlalchemy_uri == "sqlite:///test.db"
|
||||||
|
|
||||||
|
db.session.delete(database)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
def test_import_v1_database_multiple(self):
|
||||||
|
"""Test that a database can be imported multiple times"""
|
||||||
|
num_databases = db.session.query(Database).count()
|
||||||
|
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
|
||||||
|
# import twice
|
||||||
|
command.run()
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
database = (
|
||||||
|
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert database.allow_csv_upload
|
||||||
|
|
||||||
|
# update allow_csv_upload to False
|
||||||
|
new_config = database_config.copy()
|
||||||
|
new_config["allow_csv_upload"] = False
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(new_config),
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
database = (
|
||||||
|
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert not database.allow_csv_upload
|
||||||
|
|
||||||
|
# test that only one database was created
|
||||||
|
new_num_databases = db.session.query(Database).count()
|
||||||
|
assert new_num_databases == num_databases + 1
|
||||||
|
|
||||||
|
db.session.delete(database)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
def test_import_v1_database_with_dataset(self):
|
||||||
|
"""Test that a database can be imported with datasets"""
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
database = (
|
||||||
|
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert len(database.tables) == 1
|
||||||
|
assert str(database.tables[0].uuid) == "10808100-158b-42c4-842e-f32b99d88dfb"
|
||||||
|
|
||||||
|
db.session.delete(database.tables[0])
|
||||||
|
db.session.delete(database)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
def test_import_v1_database_with_dataset_multiple(self):
|
||||||
|
"""Test that a database can be imported multiple times w/o changing datasets"""
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
dataset = (
|
||||||
|
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert dataset.offset == 66
|
||||||
|
|
||||||
|
new_config = dataset_config.copy()
|
||||||
|
new_config["offset"] = 67
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(new_config),
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
# the underlying dataset should not be modified by the second import, since
|
||||||
|
# we're importing a database, not a dataset
|
||||||
|
dataset = (
|
||||||
|
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert dataset.offset == 66
|
||||||
|
|
||||||
|
db.session.delete(dataset)
|
||||||
|
db.session.delete(dataset.database)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
def test_import_v1_database_validation(self):
|
||||||
|
"""Test different validations applied when importing a database"""
|
||||||
|
# metadata.yaml must be present
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
with pytest.raises(IncorrectVersionError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Missing metadata.yaml"
|
||||||
|
|
||||||
|
# version should be 1.0.0
|
||||||
|
contents["metadata.yaml"] = yaml.safe_dump(
|
||||||
|
{
|
||||||
|
"version": "2.0.0",
|
||||||
|
"type": "Database",
|
||||||
|
"timestamp": "2020-11-04T21:27:44.423819+00:00",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
with pytest.raises(IncorrectVersionError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Must be equal to 1.0.0."
|
||||||
|
|
||||||
|
# type should be Database
|
||||||
|
contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config)
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
with pytest.raises(CommandInvalidError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Error importing database"
|
||||||
|
assert excinfo.value.normalized_messages() == {
|
||||||
|
"metadata.yaml": {"type": ["Must be equal to Database."],}
|
||||||
|
}
|
||||||
|
|
||||||
|
# must also validate datasets
|
||||||
|
broken_config = dataset_config.copy()
|
||||||
|
del broken_config["table_name"]
|
||||||
|
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
|
||||||
|
contents["datasets/imported_dataset.yaml"] = yaml.safe_dump(broken_config)
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
with pytest.raises(CommandInvalidError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Error importing database"
|
||||||
|
assert excinfo.value.normalized_messages() == {
|
||||||
|
"datasets/imported_dataset.yaml": {
|
||||||
|
"table_name": ["Missing data for required field."],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@patch("superset.databases.commands.importers.v1.import_dataset")
|
||||||
|
def test_import_v1_rollback(self, mock_import_dataset):
|
||||||
|
"""Test than on an exception everything is rolled back"""
|
||||||
|
num_databases = db.session.query(Database).count()
|
||||||
|
|
||||||
|
# raise an exception when importing the dataset, after the database has
|
||||||
|
# already been imported
|
||||||
|
mock_import_dataset.side_effect = Exception("A wild exception appears!")
|
||||||
|
|
||||||
|
contents = {
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
|
||||||
|
"metadata.yaml": yaml.safe_dump(database_metadata_config),
|
||||||
|
}
|
||||||
|
command = ImportDatabasesCommand(contents)
|
||||||
|
with pytest.raises(Exception) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "A wild exception appears!"
|
||||||
|
|
||||||
|
# verify that the database was not added
|
||||||
|
new_num_databases = db.session.query(Database).count()
|
||||||
|
assert new_num_databases == num_databases
|
||||||
|
|
|
@ -14,18 +14,29 @@
|
||||||
# KIND, either express or implied. See the License for the
|
# KIND, either express or implied. See the License for the
|
||||||
# specific language governing permissions and limitations
|
# specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
# pylint: disable=no-self-use, invalid-name
|
||||||
|
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
from superset import security_manager
|
from superset import db, security_manager
|
||||||
|
from superset.commands.exceptions import CommandInvalidError
|
||||||
|
from superset.commands.importers.exceptions import IncorrectVersionError
|
||||||
from superset.connectors.sqla.models import SqlaTable
|
from superset.connectors.sqla.models import SqlaTable
|
||||||
from superset.datasets.commands.exceptions import DatasetNotFoundError
|
from superset.datasets.commands.exceptions import DatasetNotFoundError
|
||||||
from superset.datasets.commands.export import ExportDatasetsCommand
|
from superset.datasets.commands.export import ExportDatasetsCommand
|
||||||
from superset.utils.core import backend, get_example_database
|
from superset.datasets.commands.importers.v1 import ImportDatasetsCommand
|
||||||
|
from superset.utils.core import get_example_database
|
||||||
from tests.base_tests import SupersetTestCase
|
from tests.base_tests import SupersetTestCase
|
||||||
|
from tests.fixtures.importexport import (
|
||||||
|
database_config,
|
||||||
|
database_metadata_config,
|
||||||
|
dataset_config,
|
||||||
|
dataset_metadata_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestExportDatasetsCommand(SupersetTestCase):
|
class TestExportDatasetsCommand(SupersetTestCase):
|
||||||
|
@ -186,3 +197,149 @@ class TestExportDatasetsCommand(SupersetTestCase):
|
||||||
"version",
|
"version",
|
||||||
"database_uuid",
|
"database_uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def test_import_v1_dataset(self):
|
||||||
|
"""Test that we can import a dataset"""
|
||||||
|
contents = {
|
||||||
|
"metadata.yaml": yaml.safe_dump(dataset_metadata_config),
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
|
||||||
|
}
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
command.run()
|
||||||
|
|
||||||
|
dataset = (
|
||||||
|
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert dataset.table_name == "imported_dataset"
|
||||||
|
assert dataset.main_dttm_col is None
|
||||||
|
assert dataset.description == "This is a dataset that was exported"
|
||||||
|
assert dataset.default_endpoint == ""
|
||||||
|
assert dataset.offset == 66
|
||||||
|
assert dataset.cache_timeout == 55
|
||||||
|
assert dataset.schema == ""
|
||||||
|
assert dataset.sql == ""
|
||||||
|
assert dataset.params is None
|
||||||
|
assert dataset.template_params is None
|
||||||
|
assert dataset.filter_select_enabled
|
||||||
|
assert dataset.fetch_values_predicate is None
|
||||||
|
assert dataset.extra is None
|
||||||
|
|
||||||
|
# database is also imported
|
||||||
|
assert str(dataset.database.uuid) == "b8a1ccd3-779d-4ab7-8ad8-9ab119d7fe89"
|
||||||
|
|
||||||
|
assert len(dataset.metrics) == 1
|
||||||
|
metric = dataset.metrics[0]
|
||||||
|
assert metric.metric_name == "count"
|
||||||
|
assert metric.verbose_name == ""
|
||||||
|
assert metric.metric_type is None
|
||||||
|
assert metric.expression == "count(1)"
|
||||||
|
assert metric.description is None
|
||||||
|
assert metric.d3format is None
|
||||||
|
assert metric.extra is None
|
||||||
|
assert metric.warning_text is None
|
||||||
|
|
||||||
|
assert len(dataset.columns) == 1
|
||||||
|
column = dataset.columns[0]
|
||||||
|
assert column.column_name == "cnt"
|
||||||
|
assert column.verbose_name == "Count of something"
|
||||||
|
assert not column.is_dttm
|
||||||
|
assert column.is_active # imported columns are set to active
|
||||||
|
assert column.type == "NUMBER"
|
||||||
|
assert not column.groupby
|
||||||
|
assert column.filterable
|
||||||
|
assert column.expression == ""
|
||||||
|
assert column.description is None
|
||||||
|
assert column.python_date_format is None
|
||||||
|
|
||||||
|
db.session.delete(dataset)
|
||||||
|
db.session.delete(dataset.database)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
def test_import_v1_dataset_multiple(self):
|
||||||
|
"""Test that a dataset can be imported multiple times"""
|
||||||
|
contents = {
|
||||||
|
"metadata.yaml": yaml.safe_dump(dataset_metadata_config),
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
|
||||||
|
}
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
command.run()
|
||||||
|
command.run()
|
||||||
|
dataset = (
|
||||||
|
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert dataset.table_name == "imported_dataset"
|
||||||
|
|
||||||
|
# test that columns and metrics sync, ie, old ones not the import
|
||||||
|
# are removed
|
||||||
|
new_config = dataset_config.copy()
|
||||||
|
new_config["metrics"][0]["metric_name"] = "count2"
|
||||||
|
new_config["columns"][0]["column_name"] = "cnt2"
|
||||||
|
contents = {
|
||||||
|
"metadata.yaml": yaml.safe_dump(dataset_metadata_config),
|
||||||
|
"databases/imported_database.yaml": yaml.safe_dump(database_config),
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(new_config),
|
||||||
|
}
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
command.run()
|
||||||
|
dataset = (
|
||||||
|
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
|
||||||
|
)
|
||||||
|
assert len(dataset.metrics) == 1
|
||||||
|
assert dataset.metrics[0].metric_name == "count2"
|
||||||
|
assert len(dataset.columns) == 1
|
||||||
|
assert dataset.columns[0].column_name == "cnt2"
|
||||||
|
|
||||||
|
db.session.delete(dataset)
|
||||||
|
db.session.delete(dataset.database)
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
def test_import_v1_dataset_validation(self):
|
||||||
|
"""Test different validations applied when importing a dataset"""
|
||||||
|
# metadata.yaml must be present
|
||||||
|
contents = {
|
||||||
|
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
|
||||||
|
}
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
with pytest.raises(IncorrectVersionError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Missing metadata.yaml"
|
||||||
|
|
||||||
|
# version should be 1.0.0
|
||||||
|
contents["metadata.yaml"] = yaml.safe_dump(
|
||||||
|
{
|
||||||
|
"version": "2.0.0",
|
||||||
|
"type": "SqlaTable",
|
||||||
|
"timestamp": "2020-11-04T21:27:44.423819+00:00",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
with pytest.raises(IncorrectVersionError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Must be equal to 1.0.0."
|
||||||
|
|
||||||
|
# type should be SqlaTable
|
||||||
|
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
with pytest.raises(CommandInvalidError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Error importing dataset"
|
||||||
|
assert excinfo.value.normalized_messages() == {
|
||||||
|
"metadata.yaml": {"type": ["Must be equal to SqlaTable."],}
|
||||||
|
}
|
||||||
|
|
||||||
|
# must also validate databases
|
||||||
|
broken_config = database_config.copy()
|
||||||
|
del broken_config["database_name"]
|
||||||
|
contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config)
|
||||||
|
contents["databases/imported_database.yaml"] = yaml.safe_dump(broken_config)
|
||||||
|
command = ImportDatasetsCommand(contents)
|
||||||
|
with pytest.raises(CommandInvalidError) as excinfo:
|
||||||
|
command.run()
|
||||||
|
assert str(excinfo.value) == "Error importing dataset"
|
||||||
|
assert excinfo.value.normalized_messages() == {
|
||||||
|
"databases/imported_database.yaml": {
|
||||||
|
"database_name": ["Missing data for required field."],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,90 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
# example YAML files
|
||||||
|
database_metadata_config: Dict[str, Any] = {
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "Database",
|
||||||
|
"timestamp": "2020-11-04T21:27:44.423819+00:00",
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset_metadata_config: Dict[str, Any] = {
|
||||||
|
"version": "1.0.0",
|
||||||
|
"type": "SqlaTable",
|
||||||
|
"timestamp": "2020-11-04T21:27:44.423819+00:00",
|
||||||
|
}
|
||||||
|
|
||||||
|
database_config: Dict[str, Any] = {
|
||||||
|
"allow_csv_upload": True,
|
||||||
|
"allow_ctas": True,
|
||||||
|
"allow_cvas": True,
|
||||||
|
"allow_run_async": False,
|
||||||
|
"cache_timeout": None,
|
||||||
|
"database_name": "imported_database",
|
||||||
|
"expose_in_sqllab": True,
|
||||||
|
"extra": {},
|
||||||
|
"sqlalchemy_uri": "sqlite:///test.db",
|
||||||
|
"uuid": "b8a1ccd3-779d-4ab7-8ad8-9ab119d7fe89",
|
||||||
|
"version": "1.0.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset_config: Dict[str, Any] = {
|
||||||
|
"table_name": "imported_dataset",
|
||||||
|
"main_dttm_col": None,
|
||||||
|
"description": "This is a dataset that was exported",
|
||||||
|
"default_endpoint": "",
|
||||||
|
"offset": 66,
|
||||||
|
"cache_timeout": 55,
|
||||||
|
"schema": "",
|
||||||
|
"sql": "",
|
||||||
|
"params": None,
|
||||||
|
"template_params": None,
|
||||||
|
"filter_select_enabled": True,
|
||||||
|
"fetch_values_predicate": None,
|
||||||
|
"extra": None,
|
||||||
|
"metrics": [
|
||||||
|
{
|
||||||
|
"metric_name": "count",
|
||||||
|
"verbose_name": "",
|
||||||
|
"metric_type": None,
|
||||||
|
"expression": "count(1)",
|
||||||
|
"description": None,
|
||||||
|
"d3format": None,
|
||||||
|
"extra": None,
|
||||||
|
"warning_text": None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"column_name": "cnt",
|
||||||
|
"verbose_name": "Count of something",
|
||||||
|
"is_dttm": False,
|
||||||
|
"is_active": None,
|
||||||
|
"type": "NUMBER",
|
||||||
|
"groupby": False,
|
||||||
|
"filterable": True,
|
||||||
|
"expression": "",
|
||||||
|
"description": None,
|
||||||
|
"python_date_format": None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"version": "1.0.0",
|
||||||
|
"uuid": "10808100-158b-42c4-842e-f32b99d88dfb",
|
||||||
|
"database_uuid": "b8a1ccd3-779d-4ab7-8ad8-9ab119d7fe89",
|
||||||
|
}
|
Loading…
Reference in New Issue