mirror of https://github.com/apache/superset.git
feat: Add new timegrains and convert_dttm to Druid engine spec (#10160)
* feat: Add new timegrains and convert_dttm to Druid engine spec * Add TemporalType enum and fix test case * Remove DATETIME for athena (original spec)
This commit is contained in:
parent
ecb44a4243
commit
b205ce32b0
|
@ -18,6 +18,7 @@ from datetime import datetime
|
|||
from typing import Optional
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class AthenaEngineSpec(BaseEngineSpec):
|
||||
|
@ -42,9 +43,9 @@ class AthenaEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"from_iso8601_date('{dttm.date().isoformat()}')"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""from_iso8601_timestamp('{dttm.isoformat(timespec="microseconds")}')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -24,6 +24,7 @@ from sqlalchemy import literal_column
|
|||
from sqlalchemy.sql.expression import ColumnClause
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# pylint: disable=unused-import
|
||||
|
@ -72,13 +73,13 @@ class BigQueryEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
|
||||
if tt == "DATETIME":
|
||||
if tt == utils.TemporalType.DATETIME:
|
||||
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS DATETIME)"""
|
||||
if tt == "TIME":
|
||||
if tt == utils.TemporalType.TIME:
|
||||
return f"""CAST('{dttm.strftime("%H:%M:%S.%f")}' AS TIME)"""
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS TIMESTAMP)"""
|
||||
return None
|
||||
|
||||
|
|
|
@ -18,6 +18,7 @@ from datetime import datetime
|
|||
from typing import Optional
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
||||
|
@ -46,8 +47,8 @@ class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"toDate('{dttm.date().isoformat()}')"
|
||||
if tt == "DATETIME":
|
||||
if tt == utils.TemporalType.DATETIME:
|
||||
return f"""toDateTime('{dttm.isoformat(sep=" ", timespec="seconds")}')"""
|
||||
return None
|
||||
|
|
|
@ -21,6 +21,7 @@ from urllib import parse
|
|||
from sqlalchemy.engine.url import URL
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class DrillEngineSpec(BaseEngineSpec):
|
||||
|
@ -54,9 +55,9 @@ class DrillEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"TO_DATE('{dttm.date().isoformat()}', 'yyyy-MM-dd')"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""TO_TIMESTAMP('{dttm.isoformat(sep=" ", timespec="seconds")}', 'yyyy-MM-dd HH:mm:ss')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -16,7 +16,8 @@
|
|||
# under the License.
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, TYPE_CHECKING
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional, TYPE_CHECKING
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
@ -41,6 +42,10 @@ class DruidEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
|||
None: "{col}",
|
||||
"PT1S": "FLOOR({col} TO SECOND)",
|
||||
"PT1M": "FLOOR({col} TO MINUTE)",
|
||||
"PT5M": "TIME_FLOOR({col}, 'PT5M')",
|
||||
"PT10M": "TIME_FLOOR({col}, 'PT10M')",
|
||||
"PT15M": "TIME_FLOOR({col}, 'PT15M')",
|
||||
"PT0.5H": "TIME_FLOOR({col}, 'PT30M')",
|
||||
"PT1H": "FLOOR({col} TO HOUR)",
|
||||
"P1D": "FLOOR({col} TO DAY)",
|
||||
"P1W": "FLOOR({col} TO WEEK)",
|
||||
|
@ -77,3 +82,12 @@ class DruidEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
|||
engine_params["connect_args"] = connect_args
|
||||
extra["engine_params"] = engine_params
|
||||
return extra
|
||||
|
||||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"CAST(TIME_PARSE('{dttm.date().isoformat()}') AS DATE)"
|
||||
if tt in (utils.TemporalType.DATETIME, utils.TemporalType.TIMESTAMP):
|
||||
return f"""TIME_PARSE('{dttm.isoformat(timespec="seconds")}')"""
|
||||
return None
|
||||
|
|
|
@ -18,6 +18,7 @@ from datetime import datetime
|
|||
from typing import Dict, Optional
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
||||
|
@ -41,6 +42,6 @@ class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-metho
|
|||
|
||||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
if target_type.upper() == "DATETIME":
|
||||
if target_type.upper() == utils.TemporalType.DATETIME:
|
||||
return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)"""
|
||||
return None
|
||||
|
|
|
@ -19,6 +19,7 @@ from typing import Optional
|
|||
|
||||
from superset.db_engine_specs.base import LimitMethod
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class HanaEngineSpec(PostgresBaseEngineSpec):
|
||||
|
@ -43,8 +44,8 @@ class HanaEngineSpec(PostgresBaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""TO_TIMESTAMP('{dttm.isoformat(timespec="microseconds")}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
|
|
@ -198,9 +198,9 @@ class HiveEngineSpec(PrestoEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""CAST('{dttm.isoformat(sep=" ", timespec="microseconds")}' AS TIMESTAMP)""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -20,6 +20,7 @@ from typing import List, Optional
|
|||
from sqlalchemy.engine.reflection import Inspector
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class ImpalaEngineSpec(BaseEngineSpec):
|
||||
|
@ -45,9 +46,9 @@ class ImpalaEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS TIMESTAMP)"""
|
||||
return None
|
||||
|
||||
|
|
|
@ -18,6 +18,7 @@ from datetime import datetime
|
|||
from typing import Optional
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
||||
|
@ -42,8 +43,8 @@ class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""CAST('{dttm.isoformat(sep=" ", timespec="seconds")}' AS TIMESTAMP)""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
|
|
@ -22,6 +22,7 @@ from typing import Any, List, Optional, Tuple, TYPE_CHECKING
|
|||
from sqlalchemy.types import String, TypeEngine, UnicodeText
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod
|
||||
from superset.utils import core as utils
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.models.core import Database # pylint: disable=unused-import
|
||||
|
@ -57,11 +58,11 @@ class MssqlEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"CONVERT(DATE, '{dttm.date().isoformat()}', 23)"
|
||||
if tt == "DATETIME":
|
||||
if tt == utils.TemporalType.DATETIME:
|
||||
return f"""CONVERT(DATETIME, '{dttm.isoformat(timespec="milliseconds")}', 126)""" # pylint: disable=line-too-long
|
||||
if tt == "SMALLDATETIME":
|
||||
if tt == utils.TemporalType.SMALLDATETIME:
|
||||
return f"""CONVERT(SMALLDATETIME, '{dttm.isoformat(sep=" ", timespec="seconds")}', 20)""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -21,6 +21,7 @@ from urllib import parse
|
|||
from sqlalchemy.engine.url import URL
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class MySQLEngineSpec(BaseEngineSpec):
|
||||
|
@ -51,9 +52,9 @@ class MySQLEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"STR_TO_DATE('{dttm.date().isoformat()}', '%Y-%m-%d')"
|
||||
if tt == "DATETIME":
|
||||
if tt == utils.TemporalType.DATETIME:
|
||||
return f"""STR_TO_DATE('{dttm.isoformat(sep=" ", timespec="microseconds")}', '%Y-%m-%d %H:%i:%s.%f')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -18,6 +18,7 @@ from datetime import datetime
|
|||
from typing import Optional
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod
|
||||
from superset.utils import core as utils
|
||||
|
||||
|
||||
class OracleEngineSpec(BaseEngineSpec):
|
||||
|
@ -41,11 +42,11 @@ class OracleEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
|
||||
if tt == "DATETIME":
|
||||
if tt == utils.TemporalType.DATETIME:
|
||||
return f"""TO_DATE('{dttm.isoformat(timespec="seconds")}', 'YYYY-MM-DD"T"HH24:MI:SS')""" # pylint: disable=line-too-long
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""TO_TIMESTAMP('{dttm.isoformat(timespec="microseconds")}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -21,6 +21,7 @@ from pytz import _FixedOffset # type: ignore
|
|||
from sqlalchemy.dialects.postgresql.base import PGInspector
|
||||
|
||||
from superset.db_engine_specs.base import BaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# pylint: disable=unused-import
|
||||
|
@ -79,8 +80,8 @@ class PostgresEngineSpec(PostgresBaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""TO_TIMESTAMP('{dttm.isoformat(sep=" ", timespec="microseconds")}', 'YYYY-MM-DD HH24:MI:SS.US')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
|
|
@ -534,9 +534,9 @@ class PrestoEngineSpec(BaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"""from_iso8601_date('{dttm.date().isoformat()}')"""
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""from_iso8601_timestamp('{dttm.isoformat(timespec="microseconds")}')""" # pylint: disable=line-too-long
|
||||
return None
|
||||
|
||||
|
|
|
@ -22,6 +22,7 @@ from urllib import parse
|
|||
from sqlalchemy.engine.url import URL
|
||||
|
||||
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
|
||||
from superset.utils import core as utils
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.models.core import Database # pylint: disable=unused-import
|
||||
|
@ -74,11 +75,11 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec):
|
|||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
tt = target_type.upper()
|
||||
if tt == "DATE":
|
||||
if tt == utils.TemporalType.DATE:
|
||||
return f"TO_DATE('{dttm.date().isoformat()}')"
|
||||
if tt == "DATETIME":
|
||||
if tt == utils.TemporalType.DATETIME:
|
||||
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS DATETIME)"""
|
||||
if tt == "TIMESTAMP":
|
||||
if tt == utils.TemporalType.TIMESTAMP:
|
||||
return f"""TO_TIMESTAMP('{dttm.isoformat(timespec="microseconds")}')"""
|
||||
return None
|
||||
|
||||
|
|
|
@ -75,7 +75,8 @@ class SqliteEngineSpec(BaseEngineSpec):
|
|||
|
||||
@classmethod
|
||||
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
|
||||
if target_type.upper() == "TEXT":
|
||||
tt = target_type.upper()
|
||||
if tt == utils.TemporalType.TEXT:
|
||||
return f"""'{dttm.isoformat(sep=" ", timespec="microseconds")}'"""
|
||||
return None
|
||||
|
||||
|
|
|
@ -1405,3 +1405,16 @@ class ChartDataResultFormat(str, Enum):
|
|||
|
||||
CSV = "csv"
|
||||
JSON = "json"
|
||||
|
||||
|
||||
class TemporalType(str, Enum):
|
||||
"""
|
||||
Supported temporal types
|
||||
"""
|
||||
|
||||
DATE = "DATE"
|
||||
DATETIME = "DATETIME"
|
||||
SMALLDATETIME = "SMALLDATETIME"
|
||||
TEXT = "TEXT"
|
||||
TIME = "TIME"
|
||||
TIMESTAMP = "TIMESTAMP"
|
||||
|
|
|
@ -0,0 +1,56 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
from sqlalchemy import column
|
||||
|
||||
from superset.db_engine_specs.druid import DruidEngineSpec
|
||||
from tests.db_engine_specs.base_tests import DbEngineSpecTestCase
|
||||
|
||||
|
||||
class DruidTestCase(DbEngineSpecTestCase):
|
||||
def test_convert_dttm(self):
|
||||
dttm = self.get_dttm()
|
||||
|
||||
self.assertEqual(
|
||||
DruidEngineSpec.convert_dttm("DATETIME", dttm),
|
||||
"TIME_PARSE('2019-01-02T03:04:05')",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
DruidEngineSpec.convert_dttm("TIMESTAMP", dttm),
|
||||
"TIME_PARSE('2019-01-02T03:04:05')",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
DruidEngineSpec.convert_dttm("DATE", dttm),
|
||||
"CAST(TIME_PARSE('2019-01-02') AS DATE)",
|
||||
)
|
||||
|
||||
def test_timegrain_expressions(self):
|
||||
"""
|
||||
DB Eng Specs (druid): Test time grain expressions
|
||||
"""
|
||||
col = "__time"
|
||||
sqla_col = column(col)
|
||||
test_cases = {
|
||||
"PT1S": f"FLOOR({col} TO SECOND)",
|
||||
"PT5M": f"TIME_FLOOR({col}, 'PT5M')",
|
||||
}
|
||||
for grain, expected in test_cases.items():
|
||||
actual = DruidEngineSpec.get_timestamp_expr(
|
||||
col=sqla_col, pdf=None, time_grain=grain
|
||||
)
|
||||
self.assertEqual(str(actual), expected)
|
Loading…
Reference in New Issue