mirror of https://github.com/apache/superset.git
refactor: from superset.utils.core break down date_parser (#12408)
This commit is contained in:
parent
321444bfd5
commit
9b0e6d0cc0
|
@ -28,12 +28,8 @@ from superset import app, is_feature_enabled
|
||||||
from superset.exceptions import QueryObjectValidationError
|
from superset.exceptions import QueryObjectValidationError
|
||||||
from superset.typing import Metric
|
from superset.typing import Metric
|
||||||
from superset.utils import pandas_postprocessing
|
from superset.utils import pandas_postprocessing
|
||||||
from superset.utils.core import (
|
from superset.utils.core import DTTM_ALIAS, json_int_dttm_ser
|
||||||
DTTM_ALIAS,
|
from superset.utils.date_parser import get_since_until, parse_human_timedelta
|
||||||
get_since_until,
|
|
||||||
json_int_dttm_ser,
|
|
||||||
parse_human_timedelta,
|
|
||||||
)
|
|
||||||
from superset.views.utils import get_time_range_endpoints
|
from superset.views.utils import get_time_range_endpoints
|
||||||
|
|
||||||
config = app.config
|
config = app.config
|
||||||
|
|
|
@ -57,6 +57,7 @@ from superset.models.core import Database
|
||||||
from superset.models.helpers import AuditMixinNullable, ImportExportMixin, QueryResult
|
from superset.models.helpers import AuditMixinNullable, ImportExportMixin, QueryResult
|
||||||
from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict
|
from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict
|
||||||
from superset.utils import core as utils
|
from superset.utils import core as utils
|
||||||
|
from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import requests
|
import requests
|
||||||
|
@ -777,7 +778,7 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
granularity["timeZone"] = timezone
|
granularity["timeZone"] = timezone
|
||||||
|
|
||||||
if origin:
|
if origin:
|
||||||
dttm = utils.parse_human_datetime(origin)
|
dttm = parse_human_datetime(origin)
|
||||||
assert dttm
|
assert dttm
|
||||||
granularity["origin"] = dttm.isoformat()
|
granularity["origin"] = dttm.isoformat()
|
||||||
|
|
||||||
|
@ -795,7 +796,7 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
else:
|
else:
|
||||||
granularity["type"] = "duration"
|
granularity["type"] = "duration"
|
||||||
granularity["duration"] = (
|
granularity["duration"] = (
|
||||||
utils.parse_human_timedelta(period_name).total_seconds() # type: ignore
|
parse_human_timedelta(period_name).total_seconds() # type: ignore
|
||||||
* 1000
|
* 1000
|
||||||
)
|
)
|
||||||
return granularity
|
return granularity
|
||||||
|
@ -938,7 +939,7 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
)
|
)
|
||||||
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
|
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
|
||||||
if self.fetch_values_from:
|
if self.fetch_values_from:
|
||||||
from_dttm = utils.parse_human_datetime(self.fetch_values_from)
|
from_dttm = parse_human_datetime(self.fetch_values_from)
|
||||||
assert from_dttm
|
assert from_dttm
|
||||||
else:
|
else:
|
||||||
from_dttm = datetime(1970, 1, 1)
|
from_dttm = datetime(1970, 1, 1)
|
||||||
|
@ -1426,7 +1427,7 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
time_offset = DruidDatasource.time_offset(query_obj["granularity"])
|
time_offset = DruidDatasource.time_offset(query_obj["granularity"])
|
||||||
|
|
||||||
def increment_timestamp(ts: str) -> datetime:
|
def increment_timestamp(ts: str) -> datetime:
|
||||||
dt = utils.parse_human_datetime(ts).replace(tzinfo=DRUID_TZ)
|
dt = parse_human_datetime(ts).replace(tzinfo=DRUID_TZ)
|
||||||
return dt + timedelta(milliseconds=time_offset)
|
return dt + timedelta(milliseconds=time_offset)
|
||||||
|
|
||||||
if DTTM_ALIAS in df.columns and time_offset:
|
if DTTM_ALIAS in df.columns and time_offset:
|
||||||
|
|
|
@ -33,7 +33,7 @@ from sqlalchemy import Column, Integer, String, Text
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
|
|
||||||
from superset import db
|
from superset import db
|
||||||
from superset.utils.core import parse_human_timedelta
|
from superset.utils.date_parser import parse_human_timedelta
|
||||||
|
|
||||||
revision = "3dda56f1c4c6"
|
revision = "3dda56f1c4c6"
|
||||||
down_revision = "bddc498dd179"
|
down_revision = "bddc498dd179"
|
||||||
|
|
|
@ -31,7 +31,7 @@ from superset.models.core import Log
|
||||||
from superset.models.dashboard import Dashboard
|
from superset.models.dashboard import Dashboard
|
||||||
from superset.models.slice import Slice
|
from superset.models.slice import Slice
|
||||||
from superset.models.tags import Tag, TaggedObject
|
from superset.models.tags import Tag, TaggedObject
|
||||||
from superset.utils.core import parse_human_datetime
|
from superset.utils.date_parser import parse_human_datetime
|
||||||
from superset.views.utils import build_extra_filters
|
from superset.views.utils import build_extra_filters
|
||||||
|
|
||||||
logger = get_task_logger(__name__)
|
logger = get_task_logger(__name__)
|
||||||
|
|
|
@ -15,7 +15,6 @@
|
||||||
# specific language governing permissions and limitations
|
# specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
"""Utility functions used across Superset"""
|
"""Utility functions used across Superset"""
|
||||||
import calendar
|
|
||||||
import decimal
|
import decimal
|
||||||
import errno
|
import errno
|
||||||
import functools
|
import functools
|
||||||
|
@ -39,7 +38,6 @@ from email.mime.multipart import MIMEMultipart
|
||||||
from email.mime.text import MIMEText
|
from email.mime.text import MIMEText
|
||||||
from email.utils import formatdate
|
from email.utils import formatdate
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from time import struct_time
|
|
||||||
from timeit import default_timer
|
from timeit import default_timer
|
||||||
from types import TracebackType
|
from types import TracebackType
|
||||||
from typing import (
|
from typing import (
|
||||||
|
@ -65,29 +63,14 @@ import bleach
|
||||||
import markdown as md
|
import markdown as md
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import parsedatetime
|
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
from cryptography import x509
|
from cryptography import x509
|
||||||
from cryptography.hazmat.backends import default_backend
|
from cryptography.hazmat.backends import default_backend
|
||||||
from cryptography.hazmat.backends.openssl.x509 import _Certificate
|
from cryptography.hazmat.backends.openssl.x509 import _Certificate
|
||||||
from dateutil.parser import parse
|
|
||||||
from dateutil.relativedelta import relativedelta
|
|
||||||
from flask import current_app, flash, g, Markup, render_template
|
from flask import current_app, flash, g, Markup, render_template
|
||||||
from flask_appbuilder import SQLA
|
from flask_appbuilder import SQLA
|
||||||
from flask_appbuilder.security.sqla.models import Role, User
|
from flask_appbuilder.security.sqla.models import Role, User
|
||||||
from flask_babel import gettext as __, lazy_gettext as _
|
from flask_babel import gettext as __
|
||||||
from holidays import CountryHoliday
|
|
||||||
from pyparsing import (
|
|
||||||
CaselessKeyword,
|
|
||||||
Forward,
|
|
||||||
Group,
|
|
||||||
Optional as ppOptional,
|
|
||||||
ParseException,
|
|
||||||
ParseResults,
|
|
||||||
pyparsing_common,
|
|
||||||
quotedString,
|
|
||||||
Suppress,
|
|
||||||
)
|
|
||||||
from sqlalchemy import event, exc, select, Text
|
from sqlalchemy import event, exc, select, Text
|
||||||
from sqlalchemy.dialects.mysql import MEDIUMTEXT
|
from sqlalchemy.dialects.mysql import MEDIUMTEXT
|
||||||
from sqlalchemy.engine import Connection, Engine
|
from sqlalchemy.engine import Connection, Engine
|
||||||
|
@ -443,58 +426,6 @@ def list_minus(l: List[Any], minus: List[Any]) -> List[Any]:
|
||||||
return [o for o in l if o not in minus]
|
return [o for o in l if o not in minus]
|
||||||
|
|
||||||
|
|
||||||
def parse_human_datetime(human_readable: str) -> datetime:
|
|
||||||
"""
|
|
||||||
Returns ``datetime.datetime`` from human readable strings
|
|
||||||
|
|
||||||
>>> from datetime import date, timedelta
|
|
||||||
>>> from dateutil.relativedelta import relativedelta
|
|
||||||
>>> parse_human_datetime('2015-04-03')
|
|
||||||
datetime.datetime(2015, 4, 3, 0, 0)
|
|
||||||
>>> parse_human_datetime('2/3/1969')
|
|
||||||
datetime.datetime(1969, 2, 3, 0, 0)
|
|
||||||
>>> parse_human_datetime('now') <= datetime.now()
|
|
||||||
True
|
|
||||||
>>> parse_human_datetime('yesterday') <= datetime.now()
|
|
||||||
True
|
|
||||||
>>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
|
|
||||||
True
|
|
||||||
>>> year_ago_1 = parse_human_datetime('one year ago').date()
|
|
||||||
>>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date()
|
|
||||||
>>> year_ago_1 == year_ago_2
|
|
||||||
True
|
|
||||||
>>> year_after_1 = parse_human_datetime('2 years after').date()
|
|
||||||
>>> year_after_2 = (datetime.now() + relativedelta(years=2)).date()
|
|
||||||
>>> year_after_1 == year_after_2
|
|
||||||
True
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
dttm = parse(human_readable)
|
|
||||||
except Exception: # pylint: disable=broad-except
|
|
||||||
try:
|
|
||||||
cal = parsedatetime.Calendar()
|
|
||||||
parsed_dttm, parsed_flags = cal.parseDT(human_readable)
|
|
||||||
# when time is not extracted, we 'reset to midnight'
|
|
||||||
if parsed_flags & 2 == 0:
|
|
||||||
parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
|
|
||||||
dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
|
|
||||||
except Exception as ex:
|
|
||||||
logger.exception(ex)
|
|
||||||
raise ValueError("Couldn't parse date string [{}]".format(human_readable))
|
|
||||||
return dttm
|
|
||||||
|
|
||||||
|
|
||||||
def dttm_from_timetuple(date_: struct_time) -> datetime:
|
|
||||||
return datetime(
|
|
||||||
date_.tm_year,
|
|
||||||
date_.tm_mon,
|
|
||||||
date_.tm_mday,
|
|
||||||
date_.tm_hour,
|
|
||||||
date_.tm_min,
|
|
||||||
date_.tm_sec,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def md5_hex(data: str) -> str:
|
def md5_hex(data: str) -> str:
|
||||||
return hashlib.md5(data.encode()).hexdigest()
|
return hashlib.md5(data.encode()).hexdigest()
|
||||||
|
|
||||||
|
@ -516,39 +447,6 @@ class DashboardEncoder(json.JSONEncoder):
|
||||||
return json.JSONEncoder(sort_keys=True).default(o)
|
return json.JSONEncoder(sort_keys=True).default(o)
|
||||||
|
|
||||||
|
|
||||||
def parse_human_timedelta(
|
|
||||||
human_readable: Optional[str], source_time: Optional[datetime] = None,
|
|
||||||
) -> timedelta:
|
|
||||||
"""
|
|
||||||
Returns ``datetime.timedelta`` from natural language time deltas
|
|
||||||
|
|
||||||
>>> parse_human_timedelta('1 day') == timedelta(days=1)
|
|
||||||
True
|
|
||||||
"""
|
|
||||||
cal = parsedatetime.Calendar()
|
|
||||||
source_dttm = dttm_from_timetuple(
|
|
||||||
source_time.timetuple() if source_time else datetime.now().timetuple()
|
|
||||||
)
|
|
||||||
modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0])
|
|
||||||
return modified_dttm - source_dttm
|
|
||||||
|
|
||||||
|
|
||||||
def parse_past_timedelta(
|
|
||||||
delta_str: str, source_time: Optional[datetime] = None
|
|
||||||
) -> timedelta:
|
|
||||||
"""
|
|
||||||
Takes a delta like '1 year' and finds the timedelta for that period in
|
|
||||||
the past, then represents that past timedelta in positive terms.
|
|
||||||
|
|
||||||
parse_human_timedelta('1 year') find the timedelta 1 year in the future.
|
|
||||||
parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
|
|
||||||
or datetime.timedelta(365).
|
|
||||||
"""
|
|
||||||
return -parse_human_timedelta(
|
|
||||||
delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class JSONEncodedDict(TypeDecorator): # pylint: disable=abstract-method
|
class JSONEncodedDict(TypeDecorator): # pylint: disable=abstract-method
|
||||||
"""Represents an immutable structure as a json-encoded string."""
|
"""Represents an immutable structure as a json-encoded string."""
|
||||||
|
|
||||||
|
@ -1254,347 +1152,6 @@ def ensure_path_exists(path: str) -> None:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
class EvalText: # pylint: disable=too-few-public-methods
|
|
||||||
def __init__(self, tokens: ParseResults) -> None:
|
|
||||||
self.value = tokens[0]
|
|
||||||
|
|
||||||
def eval(self) -> str:
|
|
||||||
# strip quotes
|
|
||||||
return self.value[1:-1]
|
|
||||||
|
|
||||||
|
|
||||||
class EvalDateTimeFunc: # pylint: disable=too-few-public-methods
|
|
||||||
def __init__(self, tokens: ParseResults) -> None:
|
|
||||||
self.value = tokens[1]
|
|
||||||
|
|
||||||
def eval(self) -> datetime:
|
|
||||||
return parse_human_datetime(self.value.eval())
|
|
||||||
|
|
||||||
|
|
||||||
class EvalDateAddFunc: # pylint: disable=too-few-public-methods
|
|
||||||
def __init__(self, tokens: ParseResults) -> None:
|
|
||||||
self.value = tokens[1]
|
|
||||||
|
|
||||||
def eval(self) -> datetime:
|
|
||||||
dttm_expression, delta, unit = self.value
|
|
||||||
dttm = dttm_expression.eval()
|
|
||||||
if unit.lower() == "quarter":
|
|
||||||
delta = delta * 3
|
|
||||||
unit = "month"
|
|
||||||
return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm)
|
|
||||||
|
|
||||||
|
|
||||||
class EvalDateTruncFunc: # pylint: disable=too-few-public-methods
|
|
||||||
def __init__(self, tokens: ParseResults) -> None:
|
|
||||||
self.value = tokens[1]
|
|
||||||
|
|
||||||
def eval(self) -> datetime:
|
|
||||||
dttm_expression, unit = self.value
|
|
||||||
dttm = dttm_expression.eval()
|
|
||||||
if unit == "year":
|
|
||||||
dttm = dttm.replace(
|
|
||||||
month=1, day=1, hour=0, minute=0, second=0, microsecond=0
|
|
||||||
)
|
|
||||||
elif unit == "month":
|
|
||||||
dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
|
||||||
elif unit == "week":
|
|
||||||
dttm = dttm - relativedelta(days=dttm.weekday())
|
|
||||||
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
||||||
elif unit == "day":
|
|
||||||
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
||||||
elif unit == "hour":
|
|
||||||
dttm = dttm.replace(minute=0, second=0, microsecond=0)
|
|
||||||
elif unit == "minute":
|
|
||||||
dttm = dttm.replace(second=0, microsecond=0)
|
|
||||||
else:
|
|
||||||
dttm = dttm.replace(microsecond=0)
|
|
||||||
return dttm
|
|
||||||
|
|
||||||
|
|
||||||
class EvalLastDayFunc: # pylint: disable=too-few-public-methods
|
|
||||||
def __init__(self, tokens: ParseResults) -> None:
|
|
||||||
self.value = tokens[1]
|
|
||||||
|
|
||||||
def eval(self) -> datetime:
|
|
||||||
dttm_expression, unit = self.value
|
|
||||||
dttm = dttm_expression.eval()
|
|
||||||
if unit == "year":
|
|
||||||
return dttm.replace(
|
|
||||||
month=12, day=31, hour=0, minute=0, second=0, microsecond=0
|
|
||||||
)
|
|
||||||
if unit == "month":
|
|
||||||
return dttm.replace(
|
|
||||||
day=calendar.monthrange(dttm.year, dttm.month)[1],
|
|
||||||
hour=0,
|
|
||||||
minute=0,
|
|
||||||
second=0,
|
|
||||||
microsecond=0,
|
|
||||||
)
|
|
||||||
# unit == "week":
|
|
||||||
mon = dttm - relativedelta(days=dttm.weekday())
|
|
||||||
mon = mon.replace(hour=0, minute=0, second=0, microsecond=0)
|
|
||||||
return mon + relativedelta(days=6)
|
|
||||||
|
|
||||||
|
|
||||||
class EvalHolidayFunc: # pylint: disable=too-few-public-methods
|
|
||||||
def __init__(self, tokens: ParseResults) -> None:
|
|
||||||
self.value = tokens[1]
|
|
||||||
|
|
||||||
def eval(self) -> datetime:
|
|
||||||
holiday = self.value[0].eval()
|
|
||||||
dttm, country = [None, None]
|
|
||||||
if len(self.value) >= 2:
|
|
||||||
dttm = self.value[1].eval()
|
|
||||||
if len(self.value) == 3:
|
|
||||||
country = self.value[2]
|
|
||||||
holiday_year = dttm.year if dttm else parse_human_datetime("today").year
|
|
||||||
country = country.eval() if country else "US"
|
|
||||||
|
|
||||||
holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False)
|
|
||||||
searched_result = holiday_lookup.get_named(holiday)
|
|
||||||
if len(searched_result) == 1:
|
|
||||||
return dttm_from_timetuple(searched_result[0].timetuple())
|
|
||||||
raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday))
|
|
||||||
|
|
||||||
|
|
||||||
@memoized()
|
|
||||||
def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals
|
|
||||||
( # pylint: disable=invalid-name
|
|
||||||
DATETIME,
|
|
||||||
DATEADD,
|
|
||||||
DATETRUNC,
|
|
||||||
LASTDAY,
|
|
||||||
HOLIDAY,
|
|
||||||
YEAR,
|
|
||||||
QUARTER,
|
|
||||||
MONTH,
|
|
||||||
WEEK,
|
|
||||||
DAY,
|
|
||||||
HOUR,
|
|
||||||
MINUTE,
|
|
||||||
SECOND,
|
|
||||||
) = map(
|
|
||||||
CaselessKeyword,
|
|
||||||
"datetime dateadd datetrunc lastday holiday "
|
|
||||||
"year quarter month week day hour minute second".split(),
|
|
||||||
)
|
|
||||||
lparen, rparen, comma = map(Suppress, "(),")
|
|
||||||
int_operand = pyparsing_common.signed_integer().setName("int_operand")
|
|
||||||
text_operand = quotedString.setName("text_operand").setParseAction(EvalText)
|
|
||||||
|
|
||||||
# allow expression to be used recursively
|
|
||||||
datetime_func = Forward().setName("datetime")
|
|
||||||
dateadd_func = Forward().setName("dateadd")
|
|
||||||
datetrunc_func = Forward().setName("datetrunc")
|
|
||||||
lastday_func = Forward().setName("lastday")
|
|
||||||
holiday_func = Forward().setName("holiday")
|
|
||||||
date_expr = (
|
|
||||||
datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func
|
|
||||||
)
|
|
||||||
|
|
||||||
datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction(
|
|
||||||
EvalDateTimeFunc
|
|
||||||
)
|
|
||||||
dateadd_func <<= (
|
|
||||||
DATEADD
|
|
||||||
+ lparen
|
|
||||||
+ Group(
|
|
||||||
date_expr
|
|
||||||
+ comma
|
|
||||||
+ int_operand
|
|
||||||
+ comma
|
|
||||||
+ (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
|
|
||||||
+ ppOptional(comma)
|
|
||||||
)
|
|
||||||
+ rparen
|
|
||||||
).setParseAction(EvalDateAddFunc)
|
|
||||||
datetrunc_func <<= (
|
|
||||||
DATETRUNC
|
|
||||||
+ lparen
|
|
||||||
+ Group(
|
|
||||||
date_expr
|
|
||||||
+ comma
|
|
||||||
+ (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
|
|
||||||
+ ppOptional(comma)
|
|
||||||
)
|
|
||||||
+ rparen
|
|
||||||
).setParseAction(EvalDateTruncFunc)
|
|
||||||
lastday_func <<= (
|
|
||||||
LASTDAY
|
|
||||||
+ lparen
|
|
||||||
+ Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma))
|
|
||||||
+ rparen
|
|
||||||
).setParseAction(EvalLastDayFunc)
|
|
||||||
holiday_func <<= (
|
|
||||||
HOLIDAY
|
|
||||||
+ lparen
|
|
||||||
+ Group(
|
|
||||||
text_operand
|
|
||||||
+ ppOptional(comma)
|
|
||||||
+ ppOptional(date_expr)
|
|
||||||
+ ppOptional(comma)
|
|
||||||
+ ppOptional(text_operand)
|
|
||||||
+ ppOptional(comma)
|
|
||||||
)
|
|
||||||
+ rparen
|
|
||||||
).setParseAction(EvalHolidayFunc)
|
|
||||||
|
|
||||||
return date_expr
|
|
||||||
|
|
||||||
|
|
||||||
def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]:
|
|
||||||
if datetime_expression:
|
|
||||||
try:
|
|
||||||
return datetime_parser().parseString(datetime_expression)[0].eval()
|
|
||||||
except ParseException as error:
|
|
||||||
raise ValueError(error)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
|
|
||||||
def get_since_until(
|
|
||||||
time_range: Optional[str] = None,
|
|
||||||
since: Optional[str] = None,
|
|
||||||
until: Optional[str] = None,
|
|
||||||
time_shift: Optional[str] = None,
|
|
||||||
relative_start: Optional[str] = None,
|
|
||||||
relative_end: Optional[str] = None,
|
|
||||||
) -> Tuple[Optional[datetime], Optional[datetime]]:
|
|
||||||
"""Return `since` and `until` date time tuple from string representations of
|
|
||||||
time_range, since, until and time_shift.
|
|
||||||
|
|
||||||
This functiom supports both reading the keys separately (from `since` and
|
|
||||||
`until`), as well as the new `time_range` key. Valid formats are:
|
|
||||||
|
|
||||||
- ISO 8601
|
|
||||||
- X days/years/hours/day/year/weeks
|
|
||||||
- X days/years/hours/day/year/weeks ago
|
|
||||||
- X days/years/hours/day/year/weeks from now
|
|
||||||
- freeform
|
|
||||||
|
|
||||||
Additionally, for `time_range` (these specify both `since` and `until`):
|
|
||||||
|
|
||||||
- Last day
|
|
||||||
- Last week
|
|
||||||
- Last month
|
|
||||||
- Last quarter
|
|
||||||
- Last year
|
|
||||||
- No filter
|
|
||||||
- Last X seconds/minutes/hours/days/weeks/months/years
|
|
||||||
- Next X seconds/minutes/hours/days/weeks/months/years
|
|
||||||
|
|
||||||
"""
|
|
||||||
separator = " : "
|
|
||||||
_relative_start = relative_start if relative_start else "today"
|
|
||||||
_relative_end = relative_end if relative_end else "today"
|
|
||||||
|
|
||||||
if time_range == "No filter":
|
|
||||||
return None, None
|
|
||||||
|
|
||||||
if time_range and time_range.startswith("Last") and separator not in time_range:
|
|
||||||
time_range = time_range + separator + _relative_end
|
|
||||||
|
|
||||||
if time_range and time_range.startswith("Next") and separator not in time_range:
|
|
||||||
time_range = _relative_start + separator + time_range
|
|
||||||
|
|
||||||
if (
|
|
||||||
time_range
|
|
||||||
and time_range.startswith("previous calendar week")
|
|
||||||
and separator not in time_range
|
|
||||||
):
|
|
||||||
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long
|
|
||||||
if (
|
|
||||||
time_range
|
|
||||||
and time_range.startswith("previous calendar month")
|
|
||||||
and separator not in time_range
|
|
||||||
):
|
|
||||||
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long
|
|
||||||
if (
|
|
||||||
time_range
|
|
||||||
and time_range.startswith("previous calendar year")
|
|
||||||
and separator not in time_range
|
|
||||||
):
|
|
||||||
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long
|
|
||||||
|
|
||||||
if time_range and separator in time_range:
|
|
||||||
time_range_lookup = [
|
|
||||||
(
|
|
||||||
r"^last\s+(day|week|month|quarter|year)$",
|
|
||||||
lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
|
|
||||||
lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long
|
|
||||||
),
|
|
||||||
(
|
|
||||||
r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
|
|
||||||
lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long
|
|
||||||
),
|
|
||||||
(
|
|
||||||
r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$",
|
|
||||||
lambda text: text,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)]
|
|
||||||
since_and_until: List[Optional[str]] = []
|
|
||||||
for part in since_and_until_partition:
|
|
||||||
if not part:
|
|
||||||
# if since or until is "", set as None
|
|
||||||
since_and_until.append(None)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Is it possible to match to time_range_lookup
|
|
||||||
matched = False
|
|
||||||
for pattern, fn in time_range_lookup:
|
|
||||||
result = re.search(pattern, part, re.IGNORECASE)
|
|
||||||
if result:
|
|
||||||
matched = True
|
|
||||||
# converted matched time_range to "formal time expressions"
|
|
||||||
since_and_until.append(fn(*result.groups())) # type: ignore
|
|
||||||
if not matched:
|
|
||||||
# default matched case
|
|
||||||
since_and_until.append(f"DATETIME('{part}')")
|
|
||||||
|
|
||||||
_since, _until = map(datetime_eval, since_and_until)
|
|
||||||
else:
|
|
||||||
since = since or ""
|
|
||||||
if since:
|
|
||||||
since = add_ago_to_since(since)
|
|
||||||
_since = parse_human_datetime(since) if since else None
|
|
||||||
_until = (
|
|
||||||
parse_human_datetime(until)
|
|
||||||
if until
|
|
||||||
else parse_human_datetime(_relative_end)
|
|
||||||
)
|
|
||||||
|
|
||||||
if time_shift:
|
|
||||||
time_delta = parse_past_timedelta(time_shift)
|
|
||||||
_since = _since if _since is None else (_since - time_delta)
|
|
||||||
_until = _until if _until is None else (_until - time_delta)
|
|
||||||
|
|
||||||
if _since and _until and _since > _until:
|
|
||||||
raise ValueError(_("From date cannot be larger than to date"))
|
|
||||||
|
|
||||||
return _since, _until
|
|
||||||
|
|
||||||
|
|
||||||
def add_ago_to_since(since: str) -> str:
|
|
||||||
"""
|
|
||||||
Backwards compatibility hack. Without this slices with since: 7 days will
|
|
||||||
be treated as 7 days in the future.
|
|
||||||
|
|
||||||
:param str since:
|
|
||||||
:returns: Since with ago added if necessary
|
|
||||||
:rtype: str
|
|
||||||
"""
|
|
||||||
since_words = since.split(" ")
|
|
||||||
grains = ["days", "years", "hours", "day", "year", "weeks"]
|
|
||||||
if len(since_words) == 2 and since_words[1] in grains:
|
|
||||||
since += " ago"
|
|
||||||
return since
|
|
||||||
|
|
||||||
|
|
||||||
def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name
|
def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name
|
||||||
form_data: FormData,
|
form_data: FormData,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|
|
@ -0,0 +1,469 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
import calendar
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from time import struct_time
|
||||||
|
from typing import List, Optional, Tuple
|
||||||
|
|
||||||
|
import parsedatetime
|
||||||
|
from dateutil.parser import parse
|
||||||
|
from dateutil.relativedelta import relativedelta
|
||||||
|
from flask_babel import lazy_gettext as _
|
||||||
|
from holidays import CountryHoliday
|
||||||
|
from pyparsing import (
|
||||||
|
CaselessKeyword,
|
||||||
|
Forward,
|
||||||
|
Group,
|
||||||
|
Optional as ppOptional,
|
||||||
|
ParseException,
|
||||||
|
ParseResults,
|
||||||
|
pyparsing_common,
|
||||||
|
quotedString,
|
||||||
|
Suppress,
|
||||||
|
)
|
||||||
|
|
||||||
|
from .core import memoized
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_human_datetime(human_readable: str) -> datetime:
|
||||||
|
"""
|
||||||
|
Returns ``datetime.datetime`` from human readable strings
|
||||||
|
|
||||||
|
>>> from datetime import date, timedelta
|
||||||
|
>>> from dateutil.relativedelta import relativedelta
|
||||||
|
>>> parse_human_datetime('2015-04-03')
|
||||||
|
datetime.datetime(2015, 4, 3, 0, 0)
|
||||||
|
>>> parse_human_datetime('2/3/1969')
|
||||||
|
datetime.datetime(1969, 2, 3, 0, 0)
|
||||||
|
>>> parse_human_datetime('now') <= datetime.now()
|
||||||
|
True
|
||||||
|
>>> parse_human_datetime('yesterday') <= datetime.now()
|
||||||
|
True
|
||||||
|
>>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
|
||||||
|
True
|
||||||
|
>>> year_ago_1 = parse_human_datetime('one year ago').date()
|
||||||
|
>>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date()
|
||||||
|
>>> year_ago_1 == year_ago_2
|
||||||
|
True
|
||||||
|
>>> year_after_1 = parse_human_datetime('2 years after').date()
|
||||||
|
>>> year_after_2 = (datetime.now() + relativedelta(years=2)).date()
|
||||||
|
>>> year_after_1 == year_after_2
|
||||||
|
True
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
dttm = parse(human_readable)
|
||||||
|
except Exception: # pylint: disable=broad-except
|
||||||
|
try:
|
||||||
|
cal = parsedatetime.Calendar()
|
||||||
|
parsed_dttm, parsed_flags = cal.parseDT(human_readable)
|
||||||
|
# when time is not extracted, we 'reset to midnight'
|
||||||
|
if parsed_flags & 2 == 0:
|
||||||
|
parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
|
||||||
|
dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
|
||||||
|
except Exception as ex:
|
||||||
|
logger.exception(ex)
|
||||||
|
raise ValueError("Couldn't parse date string [{}]".format(human_readable))
|
||||||
|
return dttm
|
||||||
|
|
||||||
|
|
||||||
|
def dttm_from_timetuple(date_: struct_time) -> datetime:
|
||||||
|
return datetime(
|
||||||
|
date_.tm_year,
|
||||||
|
date_.tm_mon,
|
||||||
|
date_.tm_mday,
|
||||||
|
date_.tm_hour,
|
||||||
|
date_.tm_min,
|
||||||
|
date_.tm_sec,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_human_timedelta(
|
||||||
|
human_readable: Optional[str], source_time: Optional[datetime] = None,
|
||||||
|
) -> timedelta:
|
||||||
|
"""
|
||||||
|
Returns ``datetime.timedelta`` from natural language time deltas
|
||||||
|
|
||||||
|
>>> parse_human_timedelta('1 day') == timedelta(days=1)
|
||||||
|
True
|
||||||
|
"""
|
||||||
|
cal = parsedatetime.Calendar()
|
||||||
|
source_dttm = dttm_from_timetuple(
|
||||||
|
source_time.timetuple() if source_time else datetime.now().timetuple()
|
||||||
|
)
|
||||||
|
modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0])
|
||||||
|
return modified_dttm - source_dttm
|
||||||
|
|
||||||
|
|
||||||
|
def parse_past_timedelta(
|
||||||
|
delta_str: str, source_time: Optional[datetime] = None
|
||||||
|
) -> timedelta:
|
||||||
|
"""
|
||||||
|
Takes a delta like '1 year' and finds the timedelta for that period in
|
||||||
|
the past, then represents that past timedelta in positive terms.
|
||||||
|
|
||||||
|
parse_human_timedelta('1 year') find the timedelta 1 year in the future.
|
||||||
|
parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
|
||||||
|
or datetime.timedelta(365).
|
||||||
|
"""
|
||||||
|
return -parse_human_timedelta(
|
||||||
|
delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
|
||||||
|
def get_since_until(
|
||||||
|
time_range: Optional[str] = None,
|
||||||
|
since: Optional[str] = None,
|
||||||
|
until: Optional[str] = None,
|
||||||
|
time_shift: Optional[str] = None,
|
||||||
|
relative_start: Optional[str] = None,
|
||||||
|
relative_end: Optional[str] = None,
|
||||||
|
) -> Tuple[Optional[datetime], Optional[datetime]]:
|
||||||
|
"""Return `since` and `until` date time tuple from string representations of
|
||||||
|
time_range, since, until and time_shift.
|
||||||
|
|
||||||
|
This functiom supports both reading the keys separately (from `since` and
|
||||||
|
`until`), as well as the new `time_range` key. Valid formats are:
|
||||||
|
|
||||||
|
- ISO 8601
|
||||||
|
- X days/years/hours/day/year/weeks
|
||||||
|
- X days/years/hours/day/year/weeks ago
|
||||||
|
- X days/years/hours/day/year/weeks from now
|
||||||
|
- freeform
|
||||||
|
|
||||||
|
Additionally, for `time_range` (these specify both `since` and `until`):
|
||||||
|
|
||||||
|
- Last day
|
||||||
|
- Last week
|
||||||
|
- Last month
|
||||||
|
- Last quarter
|
||||||
|
- Last year
|
||||||
|
- No filter
|
||||||
|
- Last X seconds/minutes/hours/days/weeks/months/years
|
||||||
|
- Next X seconds/minutes/hours/days/weeks/months/years
|
||||||
|
|
||||||
|
"""
|
||||||
|
separator = " : "
|
||||||
|
_relative_start = relative_start if relative_start else "today"
|
||||||
|
_relative_end = relative_end if relative_end else "today"
|
||||||
|
|
||||||
|
if time_range == "No filter":
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
if time_range and time_range.startswith("Last") and separator not in time_range:
|
||||||
|
time_range = time_range + separator + _relative_end
|
||||||
|
|
||||||
|
if time_range and time_range.startswith("Next") and separator not in time_range:
|
||||||
|
time_range = _relative_start + separator + time_range
|
||||||
|
|
||||||
|
if (
|
||||||
|
time_range
|
||||||
|
and time_range.startswith("previous calendar week")
|
||||||
|
and separator not in time_range
|
||||||
|
):
|
||||||
|
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long
|
||||||
|
if (
|
||||||
|
time_range
|
||||||
|
and time_range.startswith("previous calendar month")
|
||||||
|
and separator not in time_range
|
||||||
|
):
|
||||||
|
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long
|
||||||
|
if (
|
||||||
|
time_range
|
||||||
|
and time_range.startswith("previous calendar year")
|
||||||
|
and separator not in time_range
|
||||||
|
):
|
||||||
|
time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long
|
||||||
|
|
||||||
|
if time_range and separator in time_range:
|
||||||
|
time_range_lookup = [
|
||||||
|
(
|
||||||
|
r"^last\s+(day|week|month|quarter|year)$",
|
||||||
|
lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
|
||||||
|
lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long
|
||||||
|
),
|
||||||
|
(
|
||||||
|
r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
|
||||||
|
lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long
|
||||||
|
),
|
||||||
|
(
|
||||||
|
r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$",
|
||||||
|
lambda text: text,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)]
|
||||||
|
since_and_until: List[Optional[str]] = []
|
||||||
|
for part in since_and_until_partition:
|
||||||
|
if not part:
|
||||||
|
# if since or until is "", set as None
|
||||||
|
since_and_until.append(None)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Is it possible to match to time_range_lookup
|
||||||
|
matched = False
|
||||||
|
for pattern, fn in time_range_lookup:
|
||||||
|
result = re.search(pattern, part, re.IGNORECASE)
|
||||||
|
if result:
|
||||||
|
matched = True
|
||||||
|
# converted matched time_range to "formal time expressions"
|
||||||
|
since_and_until.append(fn(*result.groups())) # type: ignore
|
||||||
|
if not matched:
|
||||||
|
# default matched case
|
||||||
|
since_and_until.append(f"DATETIME('{part}')")
|
||||||
|
|
||||||
|
_since, _until = map(datetime_eval, since_and_until)
|
||||||
|
else:
|
||||||
|
since = since or ""
|
||||||
|
if since:
|
||||||
|
since = add_ago_to_since(since)
|
||||||
|
_since = parse_human_datetime(since) if since else None
|
||||||
|
_until = (
|
||||||
|
parse_human_datetime(until)
|
||||||
|
if until
|
||||||
|
else parse_human_datetime(_relative_end)
|
||||||
|
)
|
||||||
|
|
||||||
|
if time_shift:
|
||||||
|
time_delta = parse_past_timedelta(time_shift)
|
||||||
|
_since = _since if _since is None else (_since - time_delta)
|
||||||
|
_until = _until if _until is None else (_until - time_delta)
|
||||||
|
|
||||||
|
if _since and _until and _since > _until:
|
||||||
|
raise ValueError(_("From date cannot be larger than to date"))
|
||||||
|
|
||||||
|
return _since, _until
|
||||||
|
|
||||||
|
|
||||||
|
def add_ago_to_since(since: str) -> str:
|
||||||
|
"""
|
||||||
|
Backwards compatibility hack. Without this slices with since: 7 days will
|
||||||
|
be treated as 7 days in the future.
|
||||||
|
|
||||||
|
:param str since:
|
||||||
|
:returns: Since with ago added if necessary
|
||||||
|
:rtype: str
|
||||||
|
"""
|
||||||
|
since_words = since.split(" ")
|
||||||
|
grains = ["days", "years", "hours", "day", "year", "weeks"]
|
||||||
|
if len(since_words) == 2 and since_words[1] in grains:
|
||||||
|
since += " ago"
|
||||||
|
return since
|
||||||
|
|
||||||
|
|
||||||
|
class EvalText: # pylint: disable=too-few-public-methods
|
||||||
|
def __init__(self, tokens: ParseResults) -> None:
|
||||||
|
self.value = tokens[0]
|
||||||
|
|
||||||
|
def eval(self) -> str:
|
||||||
|
# strip quotes
|
||||||
|
return self.value[1:-1]
|
||||||
|
|
||||||
|
|
||||||
|
class EvalDateTimeFunc: # pylint: disable=too-few-public-methods
|
||||||
|
def __init__(self, tokens: ParseResults) -> None:
|
||||||
|
self.value = tokens[1]
|
||||||
|
|
||||||
|
def eval(self) -> datetime:
|
||||||
|
return parse_human_datetime(self.value.eval())
|
||||||
|
|
||||||
|
|
||||||
|
class EvalDateAddFunc: # pylint: disable=too-few-public-methods
|
||||||
|
def __init__(self, tokens: ParseResults) -> None:
|
||||||
|
self.value = tokens[1]
|
||||||
|
|
||||||
|
def eval(self) -> datetime:
|
||||||
|
dttm_expression, delta, unit = self.value
|
||||||
|
dttm = dttm_expression.eval()
|
||||||
|
if unit.lower() == "quarter":
|
||||||
|
delta = delta * 3
|
||||||
|
unit = "month"
|
||||||
|
return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm)
|
||||||
|
|
||||||
|
|
||||||
|
class EvalDateTruncFunc: # pylint: disable=too-few-public-methods
|
||||||
|
def __init__(self, tokens: ParseResults) -> None:
|
||||||
|
self.value = tokens[1]
|
||||||
|
|
||||||
|
def eval(self) -> datetime:
|
||||||
|
dttm_expression, unit = self.value
|
||||||
|
dttm = dttm_expression.eval()
|
||||||
|
if unit == "year":
|
||||||
|
dttm = dttm.replace(
|
||||||
|
month=1, day=1, hour=0, minute=0, second=0, microsecond=0
|
||||||
|
)
|
||||||
|
elif unit == "month":
|
||||||
|
dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
elif unit == "week":
|
||||||
|
dttm = dttm - relativedelta(days=dttm.weekday())
|
||||||
|
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
elif unit == "day":
|
||||||
|
dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
elif unit == "hour":
|
||||||
|
dttm = dttm.replace(minute=0, second=0, microsecond=0)
|
||||||
|
elif unit == "minute":
|
||||||
|
dttm = dttm.replace(second=0, microsecond=0)
|
||||||
|
else:
|
||||||
|
dttm = dttm.replace(microsecond=0)
|
||||||
|
return dttm
|
||||||
|
|
||||||
|
|
||||||
|
class EvalLastDayFunc: # pylint: disable=too-few-public-methods
|
||||||
|
def __init__(self, tokens: ParseResults) -> None:
|
||||||
|
self.value = tokens[1]
|
||||||
|
|
||||||
|
def eval(self) -> datetime:
|
||||||
|
dttm_expression, unit = self.value
|
||||||
|
dttm = dttm_expression.eval()
|
||||||
|
if unit == "year":
|
||||||
|
return dttm.replace(
|
||||||
|
month=12, day=31, hour=0, minute=0, second=0, microsecond=0
|
||||||
|
)
|
||||||
|
if unit == "month":
|
||||||
|
return dttm.replace(
|
||||||
|
day=calendar.monthrange(dttm.year, dttm.month)[1],
|
||||||
|
hour=0,
|
||||||
|
minute=0,
|
||||||
|
second=0,
|
||||||
|
microsecond=0,
|
||||||
|
)
|
||||||
|
# unit == "week":
|
||||||
|
mon = dttm - relativedelta(days=dttm.weekday())
|
||||||
|
mon = mon.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
return mon + relativedelta(days=6)
|
||||||
|
|
||||||
|
|
||||||
|
class EvalHolidayFunc: # pylint: disable=too-few-public-methods
|
||||||
|
def __init__(self, tokens: ParseResults) -> None:
|
||||||
|
self.value = tokens[1]
|
||||||
|
|
||||||
|
def eval(self) -> datetime:
|
||||||
|
holiday = self.value[0].eval()
|
||||||
|
dttm, country = [None, None]
|
||||||
|
if len(self.value) >= 2:
|
||||||
|
dttm = self.value[1].eval()
|
||||||
|
if len(self.value) == 3:
|
||||||
|
country = self.value[2]
|
||||||
|
holiday_year = dttm.year if dttm else parse_human_datetime("today").year
|
||||||
|
country = country.eval() if country else "US"
|
||||||
|
|
||||||
|
holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False)
|
||||||
|
searched_result = holiday_lookup.get_named(holiday)
|
||||||
|
if len(searched_result) == 1:
|
||||||
|
return dttm_from_timetuple(searched_result[0].timetuple())
|
||||||
|
raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday))
|
||||||
|
|
||||||
|
|
||||||
|
@memoized()
|
||||||
|
def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals
|
||||||
|
( # pylint: disable=invalid-name
|
||||||
|
DATETIME,
|
||||||
|
DATEADD,
|
||||||
|
DATETRUNC,
|
||||||
|
LASTDAY,
|
||||||
|
HOLIDAY,
|
||||||
|
YEAR,
|
||||||
|
QUARTER,
|
||||||
|
MONTH,
|
||||||
|
WEEK,
|
||||||
|
DAY,
|
||||||
|
HOUR,
|
||||||
|
MINUTE,
|
||||||
|
SECOND,
|
||||||
|
) = map(
|
||||||
|
CaselessKeyword,
|
||||||
|
"datetime dateadd datetrunc lastday holiday "
|
||||||
|
"year quarter month week day hour minute second".split(),
|
||||||
|
)
|
||||||
|
lparen, rparen, comma = map(Suppress, "(),")
|
||||||
|
int_operand = pyparsing_common.signed_integer().setName("int_operand")
|
||||||
|
text_operand = quotedString.setName("text_operand").setParseAction(EvalText)
|
||||||
|
|
||||||
|
# allow expression to be used recursively
|
||||||
|
datetime_func = Forward().setName("datetime")
|
||||||
|
dateadd_func = Forward().setName("dateadd")
|
||||||
|
datetrunc_func = Forward().setName("datetrunc")
|
||||||
|
lastday_func = Forward().setName("lastday")
|
||||||
|
holiday_func = Forward().setName("holiday")
|
||||||
|
date_expr = (
|
||||||
|
datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func
|
||||||
|
)
|
||||||
|
|
||||||
|
datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction(
|
||||||
|
EvalDateTimeFunc
|
||||||
|
)
|
||||||
|
dateadd_func <<= (
|
||||||
|
DATEADD
|
||||||
|
+ lparen
|
||||||
|
+ Group(
|
||||||
|
date_expr
|
||||||
|
+ comma
|
||||||
|
+ int_operand
|
||||||
|
+ comma
|
||||||
|
+ (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
|
||||||
|
+ ppOptional(comma)
|
||||||
|
)
|
||||||
|
+ rparen
|
||||||
|
).setParseAction(EvalDateAddFunc)
|
||||||
|
datetrunc_func <<= (
|
||||||
|
DATETRUNC
|
||||||
|
+ lparen
|
||||||
|
+ Group(
|
||||||
|
date_expr
|
||||||
|
+ comma
|
||||||
|
+ (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
|
||||||
|
+ ppOptional(comma)
|
||||||
|
)
|
||||||
|
+ rparen
|
||||||
|
).setParseAction(EvalDateTruncFunc)
|
||||||
|
lastday_func <<= (
|
||||||
|
LASTDAY
|
||||||
|
+ lparen
|
||||||
|
+ Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma))
|
||||||
|
+ rparen
|
||||||
|
).setParseAction(EvalLastDayFunc)
|
||||||
|
holiday_func <<= (
|
||||||
|
HOLIDAY
|
||||||
|
+ lparen
|
||||||
|
+ Group(
|
||||||
|
text_operand
|
||||||
|
+ ppOptional(comma)
|
||||||
|
+ ppOptional(date_expr)
|
||||||
|
+ ppOptional(comma)
|
||||||
|
+ ppOptional(text_operand)
|
||||||
|
+ ppOptional(comma)
|
||||||
|
)
|
||||||
|
+ rparen
|
||||||
|
).setParseAction(EvalHolidayFunc)
|
||||||
|
|
||||||
|
return date_expr
|
||||||
|
|
||||||
|
|
||||||
|
def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]:
|
||||||
|
if datetime_expression:
|
||||||
|
try:
|
||||||
|
return datetime_parser().parseString(datetime_expression)[0].eval()
|
||||||
|
except ParseException as error:
|
||||||
|
raise ValueError(error)
|
||||||
|
return None
|
|
@ -29,7 +29,7 @@ from superset.legacy import update_time_range
|
||||||
from superset.models.slice import Slice
|
from superset.models.slice import Slice
|
||||||
from superset.typing import FlaskResponse
|
from superset.typing import FlaskResponse
|
||||||
from superset.utils import core as utils
|
from superset.utils import core as utils
|
||||||
from superset.utils.core import get_since_until
|
from superset.utils.date_parser import get_since_until
|
||||||
from superset.views.base import api, BaseSupersetView, handle_api_exception
|
from superset.views.base import api, BaseSupersetView, handle_api_exception
|
||||||
|
|
||||||
get_time_range_schema = {"type": "string"}
|
get_time_range_schema = {"type": "string"}
|
||||||
|
|
|
@ -75,6 +75,7 @@ from superset.utils.core import (
|
||||||
QueryMode,
|
QueryMode,
|
||||||
to_adhoc,
|
to_adhoc,
|
||||||
)
|
)
|
||||||
|
from superset.utils.date_parser import get_since_until, parse_past_timedelta
|
||||||
from superset.utils.dates import datetime_to_epoch
|
from superset.utils.dates import datetime_to_epoch
|
||||||
from superset.utils.hashing import md5_sha_from_str
|
from superset.utils.hashing import md5_sha_from_str
|
||||||
|
|
||||||
|
@ -356,7 +357,7 @@ class BaseViz:
|
||||||
order_desc = form_data.get("order_desc", True)
|
order_desc = form_data.get("order_desc", True)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
since, until = utils.get_since_until(
|
since, until = get_since_until(
|
||||||
relative_start=relative_start,
|
relative_start=relative_start,
|
||||||
relative_end=relative_end,
|
relative_end=relative_end,
|
||||||
time_range=form_data.get("time_range"),
|
time_range=form_data.get("time_range"),
|
||||||
|
@ -367,7 +368,7 @@ class BaseViz:
|
||||||
raise QueryObjectValidationError(str(ex))
|
raise QueryObjectValidationError(str(ex))
|
||||||
|
|
||||||
time_shift = form_data.get("time_shift", "")
|
time_shift = form_data.get("time_shift", "")
|
||||||
self.time_shift = utils.parse_past_timedelta(time_shift)
|
self.time_shift = parse_past_timedelta(time_shift)
|
||||||
from_dttm = None if since is None else (since - self.time_shift)
|
from_dttm = None if since is None else (since - self.time_shift)
|
||||||
to_dttm = None if until is None else (until - self.time_shift)
|
to_dttm = None if until is None else (until - self.time_shift)
|
||||||
if from_dttm and to_dttm and from_dttm > to_dttm:
|
if from_dttm and to_dttm and from_dttm > to_dttm:
|
||||||
|
@ -1004,7 +1005,7 @@ class CalHeatmapViz(BaseViz):
|
||||||
data[metric] = values
|
data[metric] = values
|
||||||
|
|
||||||
try:
|
try:
|
||||||
start, end = utils.get_since_until(
|
start, end = get_since_until(
|
||||||
relative_start=relative_start,
|
relative_start=relative_start,
|
||||||
relative_end=relative_end,
|
relative_end=relative_end,
|
||||||
time_range=form_data.get("time_range"),
|
time_range=form_data.get("time_range"),
|
||||||
|
@ -1318,7 +1319,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
|
||||||
for option in time_compare:
|
for option in time_compare:
|
||||||
query_object = self.query_obj()
|
query_object = self.query_obj()
|
||||||
try:
|
try:
|
||||||
delta = utils.parse_past_timedelta(option)
|
delta = parse_past_timedelta(option)
|
||||||
except ValueError as ex:
|
except ValueError as ex:
|
||||||
raise QueryObjectValidationError(str(ex))
|
raise QueryObjectValidationError(str(ex))
|
||||||
query_object["inner_from_dttm"] = query_object["from_dttm"]
|
query_object["inner_from_dttm"] = query_object["from_dttm"]
|
||||||
|
|
|
@ -63,6 +63,7 @@ from superset.utils.core import (
|
||||||
merge_extra_filters,
|
merge_extra_filters,
|
||||||
to_adhoc,
|
to_adhoc,
|
||||||
)
|
)
|
||||||
|
from superset.utils.date_parser import get_since_until, parse_past_timedelta
|
||||||
|
|
||||||
import dataclasses # isort:skip
|
import dataclasses # isort:skip
|
||||||
|
|
||||||
|
@ -359,7 +360,7 @@ class BaseViz:
|
||||||
# default order direction
|
# default order direction
|
||||||
order_desc = form_data.get("order_desc", True)
|
order_desc = form_data.get("order_desc", True)
|
||||||
|
|
||||||
since, until = utils.get_since_until(
|
since, until = get_since_until(
|
||||||
relative_start=relative_start,
|
relative_start=relative_start,
|
||||||
relative_end=relative_end,
|
relative_end=relative_end,
|
||||||
time_range=form_data.get("time_range"),
|
time_range=form_data.get("time_range"),
|
||||||
|
@ -367,7 +368,7 @@ class BaseViz:
|
||||||
until=form_data.get("until"),
|
until=form_data.get("until"),
|
||||||
)
|
)
|
||||||
time_shift = form_data.get("time_shift", "")
|
time_shift = form_data.get("time_shift", "")
|
||||||
self.time_shift = utils.parse_past_timedelta(time_shift)
|
self.time_shift = parse_past_timedelta(time_shift)
|
||||||
from_dttm = None if since is None else (since - self.time_shift)
|
from_dttm = None if since is None else (since - self.time_shift)
|
||||||
to_dttm = None if until is None else (until - self.time_shift)
|
to_dttm = None if until is None else (until - self.time_shift)
|
||||||
if from_dttm and to_dttm and from_dttm > to_dttm:
|
if from_dttm and to_dttm and from_dttm > to_dttm:
|
||||||
|
@ -883,7 +884,7 @@ class CalHeatmapViz(BaseViz):
|
||||||
values[str(v / 10 ** 9)] = obj.get(metric)
|
values[str(v / 10 ** 9)] = obj.get(metric)
|
||||||
data[metric] = values
|
data[metric] = values
|
||||||
|
|
||||||
start, end = utils.get_since_until(
|
start, end = get_since_until(
|
||||||
relative_start=relative_start,
|
relative_start=relative_start,
|
||||||
relative_end=relative_end,
|
relative_end=relative_end,
|
||||||
time_range=form_data.get("time_range"),
|
time_range=form_data.get("time_range"),
|
||||||
|
@ -1265,7 +1266,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
|
||||||
|
|
||||||
for option in time_compare:
|
for option in time_compare:
|
||||||
query_object = self.query_obj()
|
query_object = self.query_obj()
|
||||||
delta = utils.parse_past_timedelta(option)
|
delta = parse_past_timedelta(option)
|
||||||
query_object["inner_from_dttm"] = query_object["from_dttm"]
|
query_object["inner_from_dttm"] = query_object["from_dttm"]
|
||||||
query_object["inner_to_dttm"] = query_object["to_dttm"]
|
query_object["inner_to_dttm"] = query_object["to_dttm"]
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,263 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from superset.utils.date_parser import (
|
||||||
|
datetime_eval,
|
||||||
|
get_since_until,
|
||||||
|
parse_human_timedelta,
|
||||||
|
parse_past_timedelta,
|
||||||
|
)
|
||||||
|
from tests.base_tests import SupersetTestCase
|
||||||
|
|
||||||
|
|
||||||
|
def mock_parse_human_datetime(s):
|
||||||
|
if s == "now":
|
||||||
|
return datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
elif s == "today":
|
||||||
|
return datetime(2016, 11, 7)
|
||||||
|
elif s == "yesterday":
|
||||||
|
return datetime(2016, 11, 6)
|
||||||
|
elif s == "tomorrow":
|
||||||
|
return datetime(2016, 11, 8)
|
||||||
|
elif s == "Last year":
|
||||||
|
return datetime(2015, 11, 7)
|
||||||
|
elif s == "Last week":
|
||||||
|
return datetime(2015, 10, 31)
|
||||||
|
elif s == "Last 5 months":
|
||||||
|
return datetime(2016, 6, 7)
|
||||||
|
elif s == "Next 5 months":
|
||||||
|
return datetime(2017, 4, 7)
|
||||||
|
elif s in ["5 days", "5 days ago"]:
|
||||||
|
return datetime(2016, 11, 2)
|
||||||
|
elif s == "2018-01-01T00:00:00":
|
||||||
|
return datetime(2018, 1, 1)
|
||||||
|
elif s == "2018-12-31T23:59:59":
|
||||||
|
return datetime(2018, 12, 31, 23, 59, 59)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDateParser(SupersetTestCase):
|
||||||
|
@patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime)
|
||||||
|
def test_get_since_until(self):
|
||||||
|
result = get_since_until()
|
||||||
|
expected = None, datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until(" : now")
|
||||||
|
expected = None, datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("yesterday : tomorrow")
|
||||||
|
expected = datetime(2016, 11, 6), datetime(2016, 11, 8)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59")
|
||||||
|
expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Last year")
|
||||||
|
expected = datetime(2015, 11, 7), datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Last quarter")
|
||||||
|
expected = datetime(2016, 8, 7), datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Last 5 months")
|
||||||
|
expected = datetime(2016, 6, 7), datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Next 5 months")
|
||||||
|
expected = datetime(2016, 11, 7), datetime(2017, 4, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until(since="5 days")
|
||||||
|
expected = datetime(2016, 11, 2), datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until(since="5 days ago", until="tomorrow")
|
||||||
|
expected = datetime(2016, 11, 2), datetime(2016, 11, 8)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day")
|
||||||
|
expected = datetime(2016, 11, 5), datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until(time_range="5 days : now")
|
||||||
|
expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Last week", relative_end="now")
|
||||||
|
expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Last week", relative_start="now")
|
||||||
|
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("Last week", relative_start="now", relative_end="now")
|
||||||
|
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("previous calendar week")
|
||||||
|
expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("previous calendar month")
|
||||||
|
expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = get_since_until("previous calendar year")
|
||||||
|
expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
get_since_until(time_range="tomorrow : yesterday")
|
||||||
|
|
||||||
|
@patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime)
|
||||||
|
def test_datetime_eval(self):
|
||||||
|
result = datetime_eval("datetime('now')")
|
||||||
|
expected = datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetime('today' )")
|
||||||
|
expected = datetime(2016, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
# Parse compact arguments spelling
|
||||||
|
result = datetime_eval("dateadd(datetime('today'),1,year,)")
|
||||||
|
expected = datetime(2017, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('today'), -2, year)")
|
||||||
|
expected = datetime(2014, 11, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('today'), 2, quarter)")
|
||||||
|
expected = datetime(2017, 5, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('today'), 3, month)")
|
||||||
|
expected = datetime(2017, 2, 7)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('today'), -3, week)")
|
||||||
|
expected = datetime(2016, 10, 17)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('today'), 3, day)")
|
||||||
|
expected = datetime(2016, 11, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('now'), 3, hour)")
|
||||||
|
expected = datetime(2016, 11, 7, 12, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('now'), 40, minute)")
|
||||||
|
expected = datetime(2016, 11, 7, 10, 10, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("dateadd(datetime('now'), -11, second)")
|
||||||
|
expected = datetime(2016, 11, 7, 9, 29, 59)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), year)")
|
||||||
|
expected = datetime(2016, 1, 1, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), month)")
|
||||||
|
expected = datetime(2016, 11, 1, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), day)")
|
||||||
|
expected = datetime(2016, 11, 7, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), week)")
|
||||||
|
expected = datetime(2016, 11, 7, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), hour)")
|
||||||
|
expected = datetime(2016, 11, 7, 9, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), minute)")
|
||||||
|
expected = datetime(2016, 11, 7, 9, 30, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("datetrunc(datetime('now'), second)")
|
||||||
|
expected = datetime(2016, 11, 7, 9, 30, 10)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("lastday(datetime('now'), year)")
|
||||||
|
expected = datetime(2016, 12, 31, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("lastday(datetime('today'), month)")
|
||||||
|
expected = datetime(2016, 11, 30, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("holiday('Christmas')")
|
||||||
|
expected = datetime(2016, 12, 25, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))")
|
||||||
|
expected = datetime(2018, 9, 3, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval(
|
||||||
|
"holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')"
|
||||||
|
)
|
||||||
|
expected = datetime(2018, 12, 26, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
result = datetime_eval(
|
||||||
|
"lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)"
|
||||||
|
)
|
||||||
|
expected = datetime(2018, 2, 28, 0, 0, 0)
|
||||||
|
self.assertEqual(result, expected)
|
||||||
|
|
||||||
|
@patch("superset.utils.date_parser.datetime")
|
||||||
|
def test_parse_human_timedelta(self, mock_datetime):
|
||||||
|
mock_datetime.now.return_value = datetime(2019, 4, 1)
|
||||||
|
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
|
||||||
|
self.assertEqual(parse_human_timedelta("now"), timedelta(0))
|
||||||
|
self.assertEqual(parse_human_timedelta("1 year"), timedelta(366))
|
||||||
|
self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365))
|
||||||
|
self.assertEqual(parse_human_timedelta(None), timedelta(0))
|
||||||
|
self.assertEqual(
|
||||||
|
parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31),
|
||||||
|
)
|
||||||
|
|
||||||
|
@patch("superset.utils.date_parser.datetime")
|
||||||
|
def test_parse_past_timedelta(self, mock_datetime):
|
||||||
|
mock_datetime.now.return_value = datetime(2019, 4, 1)
|
||||||
|
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
|
||||||
|
self.assertEqual(parse_past_timedelta("1 year"), timedelta(365))
|
||||||
|
self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365))
|
||||||
|
self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364))
|
||||||
|
self.assertEqual(parse_past_timedelta("1 month"), timedelta(31))
|
|
@ -48,7 +48,6 @@ from superset.utils.core import (
|
||||||
get_iterable,
|
get_iterable,
|
||||||
get_email_address_list,
|
get_email_address_list,
|
||||||
get_or_create_db,
|
get_or_create_db,
|
||||||
get_since_until,
|
|
||||||
get_stacktrace,
|
get_stacktrace,
|
||||||
json_int_dttm_ser,
|
json_int_dttm_ser,
|
||||||
json_iso_dttm_ser,
|
json_iso_dttm_ser,
|
||||||
|
@ -57,15 +56,12 @@ from superset.utils.core import (
|
||||||
merge_extra_filters,
|
merge_extra_filters,
|
||||||
merge_request_params,
|
merge_request_params,
|
||||||
parse_ssl_cert,
|
parse_ssl_cert,
|
||||||
parse_human_timedelta,
|
|
||||||
parse_js_uri_path_item,
|
parse_js_uri_path_item,
|
||||||
parse_past_timedelta,
|
|
||||||
split,
|
split,
|
||||||
TimeRangeEndpoint,
|
TimeRangeEndpoint,
|
||||||
validate_json,
|
validate_json,
|
||||||
zlib_compress,
|
zlib_compress,
|
||||||
zlib_decompress,
|
zlib_decompress,
|
||||||
datetime_eval,
|
|
||||||
)
|
)
|
||||||
from superset.utils import schema
|
from superset.utils import schema
|
||||||
from superset.views.utils import (
|
from superset.views.utils import (
|
||||||
|
@ -78,31 +74,6 @@ from tests.base_tests import SupersetTestCase
|
||||||
from .fixtures.certificates import ssl_certificate
|
from .fixtures.certificates import ssl_certificate
|
||||||
|
|
||||||
|
|
||||||
def mock_parse_human_datetime(s):
|
|
||||||
if s == "now":
|
|
||||||
return datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
elif s == "today":
|
|
||||||
return datetime(2016, 11, 7)
|
|
||||||
elif s == "yesterday":
|
|
||||||
return datetime(2016, 11, 6)
|
|
||||||
elif s == "tomorrow":
|
|
||||||
return datetime(2016, 11, 8)
|
|
||||||
elif s == "Last year":
|
|
||||||
return datetime(2015, 11, 7)
|
|
||||||
elif s == "Last week":
|
|
||||||
return datetime(2015, 10, 31)
|
|
||||||
elif s == "Last 5 months":
|
|
||||||
return datetime(2016, 6, 7)
|
|
||||||
elif s == "Next 5 months":
|
|
||||||
return datetime(2017, 4, 7)
|
|
||||||
elif s in ["5 days", "5 days ago"]:
|
|
||||||
return datetime(2016, 11, 2)
|
|
||||||
elif s == "2018-01-01T00:00:00":
|
|
||||||
return datetime(2018, 1, 1)
|
|
||||||
elif s == "2018-12-31T23:59:59":
|
|
||||||
return datetime(2018, 12, 31, 23, 59, 59)
|
|
||||||
|
|
||||||
|
|
||||||
def mock_to_adhoc(filt, expressionType="SIMPLE", clause="where"):
|
def mock_to_adhoc(filt, expressionType="SIMPLE", clause="where"):
|
||||||
result = {"clause": clause.upper(), "expressionType": expressionType}
|
result = {"clause": clause.upper(), "expressionType": expressionType}
|
||||||
|
|
||||||
|
@ -149,36 +120,6 @@ class TestUtils(SupersetTestCase):
|
||||||
assert isinstance(base_json_conv(uuid.uuid4()), str) is True
|
assert isinstance(base_json_conv(uuid.uuid4()), str) is True
|
||||||
assert isinstance(base_json_conv(timedelta(0)), str) is True
|
assert isinstance(base_json_conv(timedelta(0)), str) is True
|
||||||
|
|
||||||
@patch("superset.utils.core.datetime")
|
|
||||||
def test_parse_human_timedelta(self, mock_datetime):
|
|
||||||
mock_datetime.now.return_value = datetime(2019, 4, 1)
|
|
||||||
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
|
|
||||||
self.assertEqual(parse_human_timedelta("now"), timedelta(0))
|
|
||||||
self.assertEqual(parse_human_timedelta("1 year"), timedelta(366))
|
|
||||||
self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365))
|
|
||||||
self.assertEqual(parse_human_timedelta(None), timedelta(0))
|
|
||||||
self.assertEqual(
|
|
||||||
parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30),
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31),
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28),
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31),
|
|
||||||
)
|
|
||||||
|
|
||||||
@patch("superset.utils.core.datetime")
|
|
||||||
def test_parse_past_timedelta(self, mock_datetime):
|
|
||||||
mock_datetime.now.return_value = datetime(2019, 4, 1)
|
|
||||||
mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
|
|
||||||
self.assertEqual(parse_past_timedelta("1 year"), timedelta(365))
|
|
||||||
self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365))
|
|
||||||
self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364))
|
|
||||||
self.assertEqual(parse_past_timedelta("1 month"), timedelta(31))
|
|
||||||
|
|
||||||
def test_zlib_compression(self):
|
def test_zlib_compression(self):
|
||||||
json_str = '{"test": 1}'
|
json_str = '{"test": 1}'
|
||||||
blob = zlib_compress(json_str)
|
blob = zlib_compress(json_str)
|
||||||
|
@ -701,186 +642,6 @@ class TestUtils(SupersetTestCase):
|
||||||
self.assertEqual(instance.watcher, 4)
|
self.assertEqual(instance.watcher, 4)
|
||||||
self.assertEqual(result1, result8)
|
self.assertEqual(result1, result8)
|
||||||
|
|
||||||
@patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime)
|
|
||||||
def test_get_since_until(self):
|
|
||||||
result = get_since_until()
|
|
||||||
expected = None, datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until(" : now")
|
|
||||||
expected = None, datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("yesterday : tomorrow")
|
|
||||||
expected = datetime(2016, 11, 6), datetime(2016, 11, 8)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59")
|
|
||||||
expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Last year")
|
|
||||||
expected = datetime(2015, 11, 7), datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Last quarter")
|
|
||||||
expected = datetime(2016, 8, 7), datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Last 5 months")
|
|
||||||
expected = datetime(2016, 6, 7), datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Next 5 months")
|
|
||||||
expected = datetime(2016, 11, 7), datetime(2017, 4, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until(since="5 days")
|
|
||||||
expected = datetime(2016, 11, 2), datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until(since="5 days ago", until="tomorrow")
|
|
||||||
expected = datetime(2016, 11, 2), datetime(2016, 11, 8)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day")
|
|
||||||
expected = datetime(2016, 11, 5), datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until(time_range="5 days : now")
|
|
||||||
expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Last week", relative_end="now")
|
|
||||||
expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Last week", relative_start="now")
|
|
||||||
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("Last week", relative_start="now", relative_end="now")
|
|
||||||
expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("previous calendar week")
|
|
||||||
expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("previous calendar month")
|
|
||||||
expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = get_since_until("previous calendar year")
|
|
||||||
expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
get_since_until(time_range="tomorrow : yesterday")
|
|
||||||
|
|
||||||
@patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime)
|
|
||||||
def test_datetime_eval(self):
|
|
||||||
result = datetime_eval("datetime('now')")
|
|
||||||
expected = datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetime('today' )")
|
|
||||||
expected = datetime(2016, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
# Parse compact arguments spelling
|
|
||||||
result = datetime_eval("dateadd(datetime('today'),1,year,)")
|
|
||||||
expected = datetime(2017, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('today'), -2, year)")
|
|
||||||
expected = datetime(2014, 11, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('today'), 2, quarter)")
|
|
||||||
expected = datetime(2017, 5, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('today'), 3, month)")
|
|
||||||
expected = datetime(2017, 2, 7)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('today'), -3, week)")
|
|
||||||
expected = datetime(2016, 10, 17)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('today'), 3, day)")
|
|
||||||
expected = datetime(2016, 11, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('now'), 3, hour)")
|
|
||||||
expected = datetime(2016, 11, 7, 12, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('now'), 40, minute)")
|
|
||||||
expected = datetime(2016, 11, 7, 10, 10, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("dateadd(datetime('now'), -11, second)")
|
|
||||||
expected = datetime(2016, 11, 7, 9, 29, 59)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), year)")
|
|
||||||
expected = datetime(2016, 1, 1, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), month)")
|
|
||||||
expected = datetime(2016, 11, 1, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), day)")
|
|
||||||
expected = datetime(2016, 11, 7, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), week)")
|
|
||||||
expected = datetime(2016, 11, 7, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), hour)")
|
|
||||||
expected = datetime(2016, 11, 7, 9, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), minute)")
|
|
||||||
expected = datetime(2016, 11, 7, 9, 30, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("datetrunc(datetime('now'), second)")
|
|
||||||
expected = datetime(2016, 11, 7, 9, 30, 10)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("lastday(datetime('now'), year)")
|
|
||||||
expected = datetime(2016, 12, 31, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("lastday(datetime('today'), month)")
|
|
||||||
expected = datetime(2016, 11, 30, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("holiday('Christmas')")
|
|
||||||
expected = datetime(2016, 12, 25, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))")
|
|
||||||
expected = datetime(2018, 9, 3, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval(
|
|
||||||
"holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')"
|
|
||||||
)
|
|
||||||
expected = datetime(2018, 12, 26, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
result = datetime_eval(
|
|
||||||
"lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)"
|
|
||||||
)
|
|
||||||
expected = datetime(2018, 2, 28, 0, 0, 0)
|
|
||||||
self.assertEqual(result, expected)
|
|
||||||
|
|
||||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||||
def test_convert_legacy_filters_into_adhoc_where(self):
|
def test_convert_legacy_filters_into_adhoc_where(self):
|
||||||
form_data = {"where": "a = 1"}
|
form_data = {"where": "a = 1"}
|
||||||
|
|
Loading…
Reference in New Issue