pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(elasticsearch): time_zone setting does not work for cast datetime expressions (#17048) 

* fix(elasticsearch): cast does not take effect for time zone settings

* test(elasticsearch): add test

* fix(test): fix typo

* docs(elasticsearch): add annotation

* docs(elasticsearch): add time_zone desc

* docs(elasticsearch): fix typo

* refactor(db_engine): change convert_dttm signature

* fix(test): fix test

* fix(es): add try catch

* fix(test): fix caplog

* fix(test): fix typo
This commit is contained in:
aniaan 2021-11-25 17:58:44 +08:00 committed by GitHub
parent cf3f0e5b55
commit 5a1c68177e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 219 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docs/src/pages/docs/Connecting to Databases/elasticsearch.mdx
View File

@ -48,3 +48,23 @@ POST /_aliases
```
Then register your table with the alias name logstasg_all
**Time zone**
By default, Superset uses UTC time zone for elasticsearch query. If you need to specify a time zone,
please edit your Database and enter the settings of your specified time zone in the Other > ENGINE PARAMETERS:
```
{
"connect_args": {
"time_zone": "Asia/Shanghai"
}
}
```
Another issue to note about the time zone problem is that before elasticsearch7.8, if you want to convert a string into a `DATETIME` object,
you need to use the `CAST` function,but this function does not support our `time_zone` setting. So it is recommended to upgrade to the version after elasticsearch7.8.
After elasticsearch7.8, you can use the `DATETIME_PARSE` function to solve this problem.
The DATETIME_PARSE function is to support our `time_zone` setting, and here you need to fill in your elasticsearch version number in the Other > VERSION setting.
the superset will use the `DATETIME_PARSE` function for conversion.

29
superset/connectors/sqla/models.py
View File

@ -264,17 +264,23 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
def db_engine_spec(self) -> Type[BaseEngineSpec]:
return self.table.db_engine_spec
@property
def db_extra(self) -> Dict[str, Any]:
return self.table.database.get_extra()
@property
def type_generic(self) -> Optional[utils.GenericDataType]:
if self.is_dttm:
return GenericDataType.TEMPORAL
column_spec = self.db_engine_spec.get_column_spec(self.type)
column_spec = self.db_engine_spec.get_column_spec(
self.type, db_extra=self.db_extra
)
return column_spec.generic_type if column_spec else None
def get_sqla_col(self, label: Optional[str] = None) -> Column:
label = label or self.column_name
db_engine_spec = self.db_engine_spec
column_spec = db_engine_spec.get_column_spec(self.type)
column_spec = db_engine_spec.get_column_spec(self.type, db_extra=self.db_extra)
type_ = column_spec.sqla_type if column_spec else None
if self.expression:
tp = self.table.get_template_processor()
@ -332,7 +338,9 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
pdf = self.python_date_format
is_epoch = pdf in ("epoch_s", "epoch_ms")
column_spec = self.db_engine_spec.get_column_spec(self.type)
column_spec = self.db_engine_spec.get_column_spec(
self.type, db_extra=self.db_extra
)
type_ = column_spec.sqla_type if column_spec else DateTime
if not self.expression and not time_grain and not is_epoch:
sqla_col = column(self.column_name, type_=type_)
@ -357,7 +365,11 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
],
) -> str:
"""Convert datetime object to a SQL expression string"""
sql = self.db_engine_spec.convert_dttm(self.type, dttm) if self.type else None
sql = (
self.db_engine_spec.convert_dttm(self.type, dttm, db_extra=self.db_extra)
if self.type
else None
)
if sql:
return sql
@ -370,10 +382,8 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
utils.TimeRangeEndpoint.INCLUSIVE,
utils.TimeRangeEndpoint.EXCLUSIVE,
):
tf = (
self.table.database.get_extra()
.get("python_date_format_by_column_name", {})
.get(self.column_name)
tf = self.db_extra.get("python_date_format_by_column_name", {}).get(
self.column_name
)
if tf:
@ -1523,10 +1533,11 @@ class SqlaTable(Model, BaseDatasource): # pylint: disable=too-many-public-metho
value = value.item()
column_ = columns_by_name[dimension]
db_extra: Dict[str, Any] = self.database.get_extra()
if column_.type and column_.is_temporal and isinstance(value, str):
sql = self.db_engine_spec.convert_dttm(
column_.type, dateutil.parser.parse(value),
column_.type, dateutil.parser.parse(value), db_extra=db_extra
)
if sql:

4
superset/connectors/sqla/utils.py
View File

@ -57,7 +57,9 @@ def get_physical_table_metadata(
db_type = db_engine_spec.column_datatype_to_string(
col["type"], db_dialect
)
type_spec = db_engine_spec.get_column_spec(db_type)
type_spec = db_engine_spec.get_column_spec(
db_type, db_extra=database.get_extra()
)
col.update(
{
"type": db_type,

4
superset/db_engine_specs/athena.py
View File

@ -61,7 +61,9 @@ class AthenaEngineSpec(BaseEngineSpec):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"from_iso8601_date('{dttm.date().isoformat()}')"

9
superset/db_engine_specs/base.py
View File

@ -64,7 +64,6 @@ from superset.sql_parse import ParsedQuery, Table
from superset.utils import core as utils
from superset.utils.core import ColumnSpec, GenericDataType
from superset.utils.hashing import md5_sha_from_str
from superset.utils.memoized import memoized
from superset.utils.network import is_hostname_valid, is_port_open
if TYPE_CHECKING:
@ -692,13 +691,14 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
@classmethod
def convert_dttm( # pylint: disable=unused-argument
cls, target_type: str, dttm: datetime,
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
"""
Convert Python datetime object to a SQL expression
:param target_type: The target type of expression
:param dttm: The datetime object
:param db_extra: The database extra object
:return: The SQL expression
"""
return None
@ -1286,10 +1286,10 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
return parsed_query.is_select()
@classmethod
@memoized
def get_column_spec( # pylint: disable=unused-argument
cls,
native_type: Optional[str],
db_extra: Optional[Dict[str, Any]] = None,
source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE,
column_type_mappings: Tuple[
Tuple[
@ -1304,6 +1304,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
Converts native database type to sqlalchemy column type.
:param native_type: Native database typee
:param source: Type coming from the database table or cursor description
:param db_extra: The database extra object
:return: ColumnSpec object
"""
col_types = cls.get_sqla_column_type(
@ -1315,7 +1316,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
# using datetimes
if generic_type == GenericDataType.TEMPORAL:
column_type = literal_dttm_type_factory(
column_type, cls, native_type or ""
column_type, cls, native_type or "", db_extra=db_extra or {}
)
is_dttm = generic_type == GenericDataType.TEMPORAL
return ColumnSpec(

4
superset/db_engine_specs/bigquery.py
View File

@ -186,7 +186,9 @@ class BigQueryEngineSpec(BaseEngineSpec):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"

6
superset/db_engine_specs/clickhouse.py
View File

@ -16,7 +16,7 @@
# under the License.
import logging
from datetime import datetime
from typing import Dict, List, Optional, Type, TYPE_CHECKING
from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING
from urllib3.exceptions import NewConnectionError
@ -72,7 +72,9 @@ class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
return new_exception(str(exception))
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"toDate('{dttm.date().isoformat()}')"

6
superset/db_engine_specs/crate.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional, TYPE_CHECKING
from typing import Any, Dict, Optional, TYPE_CHECKING
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
@ -50,7 +50,9 @@ class CrateEngineSpec(BaseEngineSpec):
return "{col}"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.TIMESTAMP:
return f"{dttm.timestamp() * 1000}"

8
superset/db_engine_specs/databricks.py
View File

@ -16,7 +16,7 @@
# under the License.o
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.hive import HiveEngineSpec
@ -40,8 +40,10 @@ class DatabricksODBCEngineSpec(BaseEngineSpec):
_time_grain_expressions = HiveEngineSpec._time_grain_expressions
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
return HiveEngineSpec.convert_dttm(target_type, dttm)
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
return HiveEngineSpec.convert_dttm(target_type, dttm, db_extra=db_extra)
@classmethod
def epoch_to_dttm(cls) -> str:

6
superset/db_engine_specs/dremio.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
@ -43,7 +43,9 @@ class DremioEngineSpec(BaseEngineSpec):
return "TO_DATE({col})"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"

6
superset/db_engine_specs/drill.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from urllib import parse
from sqlalchemy.engine.url import URL
@ -55,7 +55,9 @@ class DrillEngineSpec(BaseEngineSpec):
return "TO_DATE({col})"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'yyyy-MM-dd')"

4
superset/db_engine_specs/druid.py
View File

@ -96,7 +96,9 @@ class DruidEngineSpec(BaseEngineSpec):
return extra
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST(TIME_PARSE('{dttm.date().isoformat()}') AS DATE)"

37
superset/db_engine_specs/elasticsearch.py
View File

@ -14,8 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import logging
from datetime import datetime
from typing import Dict, Optional, Type
from distutils.version import StrictVersion
from typing import Any, Dict, Optional, Type
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.exceptions import (
@ -25,6 +27,8 @@ from superset.db_engine_specs.exceptions import (
)
from superset.utils import core as utils
logger = logging.getLogger()
class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
engine = "elasticsearch"
@ -59,9 +63,34 @@ class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-metho
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
db_extra = db_extra or {}
if target_type.upper() == utils.TemporalType.DATETIME:
es_version = db_extra.get("version")
# The elasticsearch CAST function does not take effect for the time zone
# setting. In elasticsearch7.8 and above, we can use the DATETIME_PARSE
# function to solve this problem.
supports_dttm_parse = False
try:
if es_version:
supports_dttm_parse = StrictVersion(es_version) >= StrictVersion(
"7.8"
)
except Exception as ex: # pylint: disable=broad-except
logger.error("Unexpected error while convert es_version", exc_info=True)
logger.exception(ex)
if supports_dttm_parse:
datetime_formatted = dttm.isoformat(sep=" ", timespec="seconds")
return (
f"""DATETIME_PARSE('{datetime_formatted}', 'yyyy-MM-dd HH:mm:ss')"""
)
return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)"""
return None
@ -87,7 +116,9 @@ class OpenDistroEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
engine_name = "ElasticSearch (OpenDistro SQL)"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
if target_type.upper() == utils.TemporalType.DATETIME:
return f"""'{dttm.isoformat(timespec="seconds")}'"""
return None

6
superset/db_engine_specs/firebird.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod
from superset.utils import core as utils
@ -70,7 +70,9 @@ class FirebirdEngineSpec(BaseEngineSpec):
return "DATEADD(second, {col}, CAST('00:00:00' AS TIMESTAMP))"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.TIMESTAMP:
dttm_formatted = dttm.isoformat(sep=" ")

6
superset/db_engine_specs/firebolt.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
@ -41,7 +41,9 @@ class FireboltEngineSpec(BaseEngineSpec):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"

6
superset/db_engine_specs/hana.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from superset.db_engine_specs.base import LimitMethod
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
@ -43,7 +43,9 @@ class HanaEngineSpec(PostgresBaseEngineSpec):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"

4
superset/db_engine_specs/hive.py
View File

@ -248,7 +248,9 @@ class HiveEngineSpec(PrestoEngineSpec):
)
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"

6
superset/db_engine_specs/impala.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import List, Optional
from typing import Any, Dict, List, Optional
from sqlalchemy.engine.reflection import Inspector
@ -45,7 +45,9 @@ class ImpalaEngineSpec(BaseEngineSpec):
return "from_unixtime({col})"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"

6
superset/db_engine_specs/kylin.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional
from typing import Any, Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
@ -40,7 +40,9 @@ class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"

4
superset/db_engine_specs/mssql.py
View File

@ -103,7 +103,9 @@ class MssqlEngineSpec(BaseEngineSpec):
return "dateadd(S, {col}, '1970-01-01')"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CONVERT(DATE, '{dttm.date().isoformat()}', 23)"

5
superset/db_engine_specs/mysql.py
View File

@ -151,7 +151,9 @@ class MySQLEngineSpec(BaseEngineSpec, BasicParametersMixin):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"STR_TO_DATE('{dttm.date().isoformat()}', '%Y-%m-%d')"
@ -204,6 +206,7 @@ class MySQLEngineSpec(BaseEngineSpec, BasicParametersMixin):
def get_column_spec(
cls,
native_type: Optional[str],
db_extra: Optional[Dict[str, Any]] = None,
source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE,
column_type_mappings: Tuple[
Tuple[

6
superset/db_engine_specs/oracle.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Any, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple
from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod
from superset.utils import core as utils
@ -41,7 +41,9 @@ class OracleEngineSpec(BaseEngineSpec):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"

5
superset/db_engine_specs/postgres.py
View File

@ -242,7 +242,9 @@ class PostgresEngineSpec(PostgresBaseEngineSpec, BasicParametersMixin):
return sorted(tables)
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
@ -279,6 +281,7 @@ class PostgresEngineSpec(PostgresBaseEngineSpec, BasicParametersMixin):
def get_column_spec(
cls,
native_type: Optional[str],
db_extra: Optional[Dict[str, Any]] = None,
source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE,
column_type_mappings: Tuple[
Tuple[

5
superset/db_engine_specs/presto.py
View File

@ -743,7 +743,9 @@ class PrestoEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-metho
uri.database = database
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"""from_iso8601_date('{dttm.date().isoformat()}')"""
@ -1216,6 +1218,7 @@ class PrestoEngineSpec(BaseEngineSpec): # pylint: disable=too-many-public-metho
def get_column_spec(
cls,
native_type: Optional[str],
db_extra: Optional[Dict[str, Any]] = None,
source: utils.ColumnTypeSource = utils.ColumnTypeSource.GET_TABLE,
column_type_mappings: Tuple[
Tuple[

6
superset/db_engine_specs/rockset.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Optional, TYPE_CHECKING
from typing import Any, Dict, Optional, TYPE_CHECKING
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
@ -50,7 +50,9 @@ class RocksetEngineSpec(BaseEngineSpec):
return "{col}"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"DATE '{dttm.date().isoformat()}'"

4
superset/db_engine_specs/snowflake.py
View File

@ -130,7 +130,9 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec):
return "DATEADD(MS, {col}, '1970-01-01')"
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}')"

4
superset/db_engine_specs/sqlite.py
View File

@ -97,7 +97,9 @@ class SqliteEngineSpec(BaseEngineSpec):
raise Exception(f"Unsupported datasource_type: {datasource_type}")
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt in (utils.TemporalType.TEXT, utils.TemporalType.DATETIME):
return f"""'{dttm.isoformat(sep=" ", timespec="microseconds")}'"""

4
superset/db_engine_specs/trino.py
View File

@ -46,7 +46,9 @@ class TrinoEngineSpec(BaseEngineSpec):
}
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
def convert_dttm(
cls, target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
value = dttm.date().isoformat()

12
superset/models/sql_types/base.py
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
from datetime import datetime
from typing import Any, Callable, Type, TYPE_CHECKING
from typing import Any, Callable, Dict, Type, TYPE_CHECKING
from flask_babel import gettext as __
from sqlalchemy import types
@ -26,7 +26,10 @@ if TYPE_CHECKING:
def literal_dttm_type_factory(
sqla_type: types.TypeEngine, db_engine_spec: Type["BaseEngineSpec"], col_type: str,
sqla_type: types.TypeEngine,
db_engine_spec: Type["BaseEngineSpec"],
col_type: str,
db_extra: Dict[str, Any],
) -> types.TypeEngine:
"""
Create a custom SQLAlchemy type that supports datetime literal binds.
@ -34,6 +37,7 @@ def literal_dttm_type_factory(
:param sqla_type: Base type to extend
:param db_engine_spec: Database engine spec which supports `convert_dttm` method
:param col_type: native column type as defined in table metadata
:param db_extra: The database extra object
:return: SQLAlchemy type that supports using datetima as literal bind
"""
# pylint: disable=too-few-public-methods
@ -42,7 +46,9 @@ def literal_dttm_type_factory(
def literal_processor(self, dialect: Dialect) -> Callable[[Any], Any]:
def process(value: Any) -> Any:
if isinstance(value, datetime):
ts_expression = db_engine_spec.convert_dttm(col_type, value)
ts_expression = db_engine_spec.convert_dttm(
col_type, value, db_extra=db_extra
)
if ts_expression is None:
raise NotImplementedError(
__(

2
tests/integration_tests/db_engine_specs/base_engine_spec_tests.py
View File

@ -251,7 +251,7 @@ class TestDbEngineSpecs(TestDbEngineSpec):
def test_convert_dttm(self):
dttm = self.get_dttm()
self.assertIsNone(BaseEngineSpec.convert_dttm("", dttm))
self.assertIsNone(BaseEngineSpec.convert_dttm("", dttm, db_extra=None))
def test_pyodbc_rows_to_tuples(self):
# Test for case when pyodbc.Row is returned (odbc driver)

38
tests/integration_tests/db_engine_specs/elasticsearch_tests.py
View File

@ -16,6 +16,7 @@
# under the License.
from unittest.mock import MagicMock
import pytest
from sqlalchemy import column
from superset.db_engine_specs.elasticsearch import (
@ -26,14 +27,47 @@ from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec
class TestElasticSearchDbEngineSpec(TestDbEngineSpec):
@pytest.fixture(autouse=True)
def inject_fixtures(self, caplog):
self._caplog = caplog
def test_convert_dttm(self):
dttm = self.get_dttm()
self.assertEqual(
ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm),
ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=None),
"CAST('2019-01-02T03:04:05' AS DATETIME)",
)
def test_convert_dttm2(self):
"""
ES 7.8 and above versions need to use the DATETIME_PARSE function to
solve the time zone problem
"""
dttm = self.get_dttm()
db_extra = {"version": "7.8"}
self.assertEqual(
ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=db_extra),
"DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')",
)
def test_convert_dttm3(self):
dttm = self.get_dttm()
db_extra = {"version": 7.8}
self.assertEqual(
ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=db_extra),
"CAST('2019-01-02T03:04:05' AS DATETIME)",
)
self.assertNotEqual(
ElasticSearchEngineSpec.convert_dttm("DATETIME", dttm, db_extra=db_extra),
"DATETIME_PARSE('2019-01-02 03:04:05', 'yyyy-MM-dd HH:mm:ss')",
)
self.assertIn("Unexpected error while convert es_version", self._caplog.text)
def test_opendistro_convert_dttm(self):
"""
DB Eng Specs (opendistro): Test convert_dttm
@ -41,7 +75,7 @@ class TestElasticSearchDbEngineSpec(TestDbEngineSpec):
dttm = self.get_dttm()
self.assertEqual(
OpenDistroEngineSpec.convert_dttm("DATETIME", dttm),
OpenDistroEngineSpec.convert_dttm("DATETIME", dttm, db_extra=None),
"'2019-01-02T03:04:05'",
)

4
tests/integration_tests/model_tests.py
View File

@ -565,6 +565,8 @@ class TestSqlaTableModel(SupersetTestCase):
def test_literal_dttm_type_factory():
orig_type = DateTime()
new_type = literal_dttm_type_factory(orig_type, PostgresEngineSpec, "TIMESTAMP")
new_type = literal_dttm_type_factory(
orig_type, PostgresEngineSpec, "TIMESTAMP", db_extra={}
)
assert type(new_type).__name__ == "TemporalWrapperType"
assert str(new_type) == str(orig_type)

4
tests/integration_tests/sqla_models_tests.py
View File

@ -521,7 +521,9 @@ def test__normalize_prequery_result_type(
dimension: str,
result: Any,
) -> None:
def _convert_dttm(target_type: str, dttm: datetime) -> Optional[str]:
def _convert_dttm(
target_type: str, dttm: datetime, db_extra: Optional[Dict[str, Any]] = None
) -> Optional[str]:
if target_type.upper() == TemporalType.TIMESTAMP:
return f"""TIME_PARSE('{dttm.isoformat(timespec="seconds")}')"""