fix(presto): default unknown types to string type (#10753)

* fix(presto): default unknown types to string type

* lint
This commit is contained in:
Ville Brofeldt 2020-09-22 13:16:54 +03:00 committed by GitHub
parent 448a41a4e7
commit bd140e018a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 7 deletions

View File

@ -902,16 +902,18 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
return label_mutated return label_mutated
@classmethod @classmethod
def get_sqla_column_type(cls, type_: str) -> Optional[TypeEngine]: def get_sqla_column_type(cls, type_: Optional[str]) -> Optional[TypeEngine]:
""" """
Return a sqlalchemy native column type that corresponds to the column type Return a sqlalchemy native column type that corresponds to the column type
defined in the data source (return None to use default type inferred by defined in the data source (return None to use default type inferred by
SQLAlchemy). Override `_column_type_mappings` for specific needs SQLAlchemy). Override `column_type_mappings` for specific needs
(see MSSQL for example of NCHAR/NVARCHAR handling). (see MSSQL for example of NCHAR/NVARCHAR handling).
:param type_: Column type returned by inspector :param type_: Column type returned by inspector
:return: SqlAlchemy column type :return: SqlAlchemy column type
""" """
if not type_:
return None
for regex, sqla_type in cls.column_type_mappings: for regex, sqla_type in cls.column_type_mappings:
match = regex.match(type_) match = regex.match(type_)
if match: if match:

View File

@ -174,7 +174,9 @@ class PrestoEngineSpec(BaseEngineSpec):
return [row[0] for row in results] return [row[0] for row in results]
@classmethod @classmethod
def _create_column_info(cls, name: str, data_type: str) -> Dict[str, Any]: def _create_column_info(
cls, name: str, data_type: types.TypeEngine
) -> Dict[str, Any]:
""" """
Create column info object Create column info object
:param name: column name :param name: column name
@ -265,8 +267,11 @@ class PrestoEngineSpec(BaseEngineSpec):
# overall structural data type # overall structural data type
column_type = cls.get_sqla_column_type(field_info[1]) column_type = cls.get_sqla_column_type(field_info[1])
if column_type is None: if column_type is None:
raise NotImplementedError( column_type = types.String()
_("Unknown column type: %(col)s", col=field_info[1]) logger.info(
"Did not recognize type %s of column %s",
field_info[1],
field_info[0],
) )
if field_info[1] == "array" or field_info[1] == "row": if field_info[1] == "array" or field_info[1] == "row":
stack.append((field_info[0], field_info[1])) stack.append((field_info[0], field_info[1]))
@ -381,8 +386,11 @@ class PrestoEngineSpec(BaseEngineSpec):
# otherwise column is a basic data type # otherwise column is a basic data type
column_type = cls.get_sqla_column_type(column.Type) column_type = cls.get_sqla_column_type(column.Type)
if column_type is None: if column_type is None:
raise NotImplementedError( column_type = types.String()
_("Unknown column type: %(col)s", col=column_type) logger.info(
"Did not recognize type %s of column %s",
str(column.Type),
str(column.Column),
) )
column_info = cls._create_column_info(column.Column, column_type) column_info = cls._create_column_info(column.Column, column_type)
column_info["nullable"] = getattr(column, "Null", True) column_info["nullable"] = getattr(column, "Null", True)

View File

@ -511,3 +511,6 @@ class TestPrestoDbEngineSpec(TestDbEngineSpec):
sqla_type = PrestoEngineSpec.get_sqla_column_type("integer") sqla_type = PrestoEngineSpec.get_sqla_column_type("integer")
assert isinstance(sqla_type, types.Integer) assert isinstance(sqla_type, types.Integer)
sqla_type = PrestoEngineSpec.get_sqla_column_type(None)
assert sqla_type is None