pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Add new timegrains and convert_dttm to Druid engine spec (#10160) 

* feat: Add new timegrains and convert_dttm to Druid engine spec

* Add TemporalType enum and fix test case

* Remove DATETIME for athena (original spec)
This commit is contained in:
Ville Brofeldt 2020-06-25 12:18:37 +03:00 committed by GitHub
parent ecb44a4243
commit b205ce32b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 133 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
superset/db_engine_specs/athena.py
View File

@ -18,6 +18,7 @@ from datetime import datetime
from typing import Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class AthenaEngineSpec(BaseEngineSpec):
@ -42,9 +43,9 @@ class AthenaEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"from_iso8601_date('{dttm.date().isoformat()}')"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""from_iso8601_timestamp('{dttm.isoformat(timespec="microseconds")}')""" # pylint: disable=line-too-long
return None

9
superset/db_engine_specs/bigquery.py
View File

@ -24,6 +24,7 @@ from sqlalchemy import literal_column
from sqlalchemy.sql.expression import ColumnClause
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
if TYPE_CHECKING:
# pylint: disable=unused-import
@ -72,13 +73,13 @@ class BigQueryEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
if tt == "DATETIME":
if tt == utils.TemporalType.DATETIME:
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS DATETIME)"""
if tt == "TIME":
if tt == utils.TemporalType.TIME:
return f"""CAST('{dttm.strftime("%H:%M:%S.%f")}' AS TIME)"""
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS TIMESTAMP)"""
return None

5
superset/db_engine_specs/clickhouse.py
View File

@ -18,6 +18,7 @@ from datetime import datetime
from typing import Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
@ -46,8 +47,8 @@ class ClickHouseEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"toDate('{dttm.date().isoformat()}')"
if tt == "DATETIME":
if tt == utils.TemporalType.DATETIME:
return f"""toDateTime('{dttm.isoformat(sep=" ", timespec="seconds")}')"""
return None

5
superset/db_engine_specs/drill.py
View File

@ -21,6 +21,7 @@ from urllib import parse
from sqlalchemy.engine.url import URL
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class DrillEngineSpec(BaseEngineSpec):
@ -54,9 +55,9 @@ class DrillEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'yyyy-MM-dd')"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""TO_TIMESTAMP('{dttm.isoformat(sep=" ", timespec="seconds")}', 'yyyy-MM-dd HH:mm:ss')""" # pylint: disable=line-too-long
return None

16
superset/db_engine_specs/druid.py
View File

@ -16,7 +16,8 @@
# under the License.
import json
import logging
from typing import Any, Dict, TYPE_CHECKING
from datetime import datetime
from typing import Any, Dict, Optional, TYPE_CHECKING
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
@ -41,6 +42,10 @@ class DruidEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
None: "{col}",
"PT1S": "FLOOR({col} TO SECOND)",
"PT1M": "FLOOR({col} TO MINUTE)",
"PT5M": "TIME_FLOOR({col}, 'PT5M')",
"PT10M": "TIME_FLOOR({col}, 'PT10M')",
"PT15M": "TIME_FLOOR({col}, 'PT15M')",
"PT0.5H": "TIME_FLOOR({col}, 'PT30M')",
"PT1H": "FLOOR({col} TO HOUR)",
"P1D": "FLOOR({col} TO DAY)",
"P1W": "FLOOR({col} TO WEEK)",
@ -77,3 +82,12 @@ class DruidEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
engine_params["connect_args"] = connect_args
extra["engine_params"] = engine_params
return extra
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == utils.TemporalType.DATE:
return f"CAST(TIME_PARSE('{dttm.date().isoformat()}') AS DATE)"
if tt in (utils.TemporalType.DATETIME, utils.TemporalType.TIMESTAMP):
return f"""TIME_PARSE('{dttm.isoformat(timespec="seconds")}')"""
return None

3
superset/db_engine_specs/elasticsearch.py
View File

@ -18,6 +18,7 @@ from datetime import datetime
from typing import Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
@ -41,6 +42,6 @@ class ElasticSearchEngineSpec(BaseEngineSpec): # pylint: disable=abstract-metho
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
if target_type.upper() == "DATETIME":
if target_type.upper() == utils.TemporalType.DATETIME:
return f"""CAST('{dttm.isoformat(timespec="seconds")}' AS DATETIME)"""
return None

5
superset/db_engine_specs/hana.py
View File

@ -19,6 +19,7 @@ from typing import Optional
from superset.db_engine_specs.base import LimitMethod
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
from superset.utils import core as utils
class HanaEngineSpec(PostgresBaseEngineSpec):
@ -43,8 +44,8 @@ class HanaEngineSpec(PostgresBaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""TO_TIMESTAMP('{dttm.isoformat(timespec="microseconds")}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""" # pylint: disable=line-too-long
return None

4
superset/db_engine_specs/hive.py
View File

@ -198,9 +198,9 @@ class HiveEngineSpec(PrestoEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""CAST('{dttm.isoformat(sep=" ", timespec="microseconds")}' AS TIMESTAMP)""" # pylint: disable=line-too-long
return None

5
superset/db_engine_specs/impala.py
View File

@ -20,6 +20,7 @@ from typing import List, Optional
from sqlalchemy.engine.reflection import Inspector
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class ImpalaEngineSpec(BaseEngineSpec):
@ -45,9 +46,9 @@ class ImpalaEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS TIMESTAMP)"""
return None

5
superset/db_engine_specs/kylin.py
View File

@ -18,6 +18,7 @@ from datetime import datetime
from typing import Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
@ -42,8 +43,8 @@ class KylinEngineSpec(BaseEngineSpec): # pylint: disable=abstract-method
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"CAST('{dttm.date().isoformat()}' AS DATE)"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""CAST('{dttm.isoformat(sep=" ", timespec="seconds")}' AS TIMESTAMP)""" # pylint: disable=line-too-long
return None

7
superset/db_engine_specs/mssql.py
View File

@ -22,6 +22,7 @@ from typing import Any, List, Optional, Tuple, TYPE_CHECKING
from sqlalchemy.types import String, TypeEngine, UnicodeText
from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod
from superset.utils import core as utils
if TYPE_CHECKING:
from superset.models.core import Database # pylint: disable=unused-import
@ -57,11 +58,11 @@ class MssqlEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"CONVERT(DATE, '{dttm.date().isoformat()}', 23)"
if tt == "DATETIME":
if tt == utils.TemporalType.DATETIME:
return f"""CONVERT(DATETIME, '{dttm.isoformat(timespec="milliseconds")}', 126)""" # pylint: disable=line-too-long
if tt == "SMALLDATETIME":
if tt == utils.TemporalType.SMALLDATETIME:
return f"""CONVERT(SMALLDATETIME, '{dttm.isoformat(sep=" ", timespec="seconds")}', 20)""" # pylint: disable=line-too-long
return None

5
superset/db_engine_specs/mysql.py
View File

@ -21,6 +21,7 @@ from urllib import parse
from sqlalchemy.engine.url import URL
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
class MySQLEngineSpec(BaseEngineSpec):
@ -51,9 +52,9 @@ class MySQLEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"STR_TO_DATE('{dttm.date().isoformat()}', '%Y-%m-%d')"
if tt == "DATETIME":
if tt == utils.TemporalType.DATETIME:
return f"""STR_TO_DATE('{dttm.isoformat(sep=" ", timespec="microseconds")}', '%Y-%m-%d %H:%i:%s.%f')""" # pylint: disable=line-too-long
return None

7
superset/db_engine_specs/oracle.py
View File

@ -18,6 +18,7 @@ from datetime import datetime
from typing import Optional
from superset.db_engine_specs.base import BaseEngineSpec, LimitMethod
from superset.utils import core as utils
class OracleEngineSpec(BaseEngineSpec):
@ -41,11 +42,11 @@ class OracleEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
if tt == "DATETIME":
if tt == utils.TemporalType.DATETIME:
return f"""TO_DATE('{dttm.isoformat(timespec="seconds")}', 'YYYY-MM-DD"T"HH24:MI:SS')""" # pylint: disable=line-too-long
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""TO_TIMESTAMP('{dttm.isoformat(timespec="microseconds")}', 'YYYY-MM-DD"T"HH24:MI:SS.ff6')""" # pylint: disable=line-too-long
return None

5
superset/db_engine_specs/postgres.py
View File

@ -21,6 +21,7 @@ from pytz import _FixedOffset # type: ignore
from sqlalchemy.dialects.postgresql.base import PGInspector
from superset.db_engine_specs.base import BaseEngineSpec
from superset.utils import core as utils
if TYPE_CHECKING:
# pylint: disable=unused-import
@ -79,8 +80,8 @@ class PostgresEngineSpec(PostgresBaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}', 'YYYY-MM-DD')"
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""TO_TIMESTAMP('{dttm.isoformat(sep=" ", timespec="microseconds")}', 'YYYY-MM-DD HH24:MI:SS.US')""" # pylint: disable=line-too-long
return None

4
superset/db_engine_specs/presto.py
View File

@ -534,9 +534,9 @@ class PrestoEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"""from_iso8601_date('{dttm.date().isoformat()}')"""
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""from_iso8601_timestamp('{dttm.isoformat(timespec="microseconds")}')""" # pylint: disable=line-too-long
return None

7
superset/db_engine_specs/snowflake.py
View File

@ -22,6 +22,7 @@ from urllib import parse
from sqlalchemy.engine.url import URL
from superset.db_engine_specs.postgres import PostgresBaseEngineSpec
from superset.utils import core as utils
if TYPE_CHECKING:
from superset.models.core import Database # pylint: disable=unused-import
@ -74,11 +75,11 @@ class SnowflakeEngineSpec(PostgresBaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
tt = target_type.upper()
if tt == "DATE":
if tt == utils.TemporalType.DATE:
return f"TO_DATE('{dttm.date().isoformat()}')"
if tt == "DATETIME":
if tt == utils.TemporalType.DATETIME:
return f"""CAST('{dttm.isoformat(timespec="microseconds")}' AS DATETIME)"""
if tt == "TIMESTAMP":
if tt == utils.TemporalType.TIMESTAMP:
return f"""TO_TIMESTAMP('{dttm.isoformat(timespec="microseconds")}')"""
return None

3
superset/db_engine_specs/sqlite.py
View File

@ -75,7 +75,8 @@ class SqliteEngineSpec(BaseEngineSpec):
@classmethod
def convert_dttm(cls, target_type: str, dttm: datetime) -> Optional[str]:
if target_type.upper() == "TEXT":
tt = target_type.upper()
if tt == utils.TemporalType.TEXT:
return f"""'{dttm.isoformat(sep=" ", timespec="microseconds")}'"""
return None

13
superset/utils/core.py
View File

@ -1405,3 +1405,16 @@ class ChartDataResultFormat(str, Enum):
CSV = "csv"
JSON = "json"
class TemporalType(str, Enum):
"""
Supported temporal types
"""
DATE = "DATE"
DATETIME = "DATETIME"
SMALLDATETIME = "SMALLDATETIME"
TEXT = "TEXT"
TIME = "TIME"
TIMESTAMP = "TIMESTAMP"

56
tests/db_engine_specs/druid_tests.py Normal file
View File

@ -0,0 +1,56 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from sqlalchemy import column
from superset.db_engine_specs.druid import DruidEngineSpec
from tests.db_engine_specs.base_tests import DbEngineSpecTestCase
class DruidTestCase(DbEngineSpecTestCase):
def test_convert_dttm(self):
dttm = self.get_dttm()
self.assertEqual(
DruidEngineSpec.convert_dttm("DATETIME", dttm),
"TIME_PARSE('2019-01-02T03:04:05')",
)
self.assertEqual(
DruidEngineSpec.convert_dttm("TIMESTAMP", dttm),
"TIME_PARSE('2019-01-02T03:04:05')",
)
self.assertEqual(
DruidEngineSpec.convert_dttm("DATE", dttm),
"CAST(TIME_PARSE('2019-01-02') AS DATE)",
)
def test_timegrain_expressions(self):
"""
DB Eng Specs (druid): Test time grain expressions
"""
col = "__time"
sqla_col = column(col)
test_cases = {
"PT1S": f"FLOOR({col} TO SECOND)",
"PT5M": f"TIME_FLOOR({col}, 'PT5M')",
}
for grain, expected in test_cases.items():
actual = DruidEngineSpec.get_timestamp_expr(
col=sqla_col, pdf=None, time_grain=grain
)
self.assertEqual(str(actual), expected)