diff --git a/superset/common/query_object.py b/superset/common/query_object.py index 7aa7ef77e7..43f7fee8ed 100644 --- a/superset/common/query_object.py +++ b/superset/common/query_object.py @@ -28,12 +28,8 @@ from superset import app, is_feature_enabled from superset.exceptions import QueryObjectValidationError from superset.typing import Metric from superset.utils import pandas_postprocessing -from superset.utils.core import ( - DTTM_ALIAS, - get_since_until, - json_int_dttm_ser, - parse_human_timedelta, -) +from superset.utils.core import DTTM_ALIAS, json_int_dttm_ser +from superset.utils.date_parser import get_since_until, parse_human_timedelta from superset.views.utils import get_time_range_endpoints config = app.config diff --git a/superset/connectors/druid/models.py b/superset/connectors/druid/models.py index fe15b24735..2a7e0f96a7 100644 --- a/superset/connectors/druid/models.py +++ b/superset/connectors/druid/models.py @@ -57,6 +57,7 @@ from superset.models.core import Database from superset.models.helpers import AuditMixinNullable, ImportExportMixin, QueryResult from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict from superset.utils import core as utils +from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta try: import requests @@ -777,7 +778,7 @@ class DruidDatasource(Model, BaseDatasource): granularity["timeZone"] = timezone if origin: - dttm = utils.parse_human_datetime(origin) + dttm = parse_human_datetime(origin) assert dttm granularity["origin"] = dttm.isoformat() @@ -795,7 +796,7 @@ class DruidDatasource(Model, BaseDatasource): else: granularity["type"] = "duration" granularity["duration"] = ( - utils.parse_human_timedelta(period_name).total_seconds() # type: ignore + parse_human_timedelta(period_name).total_seconds() # type: ignore * 1000 ) return granularity @@ -938,7 +939,7 @@ class DruidDatasource(Model, BaseDatasource): ) # TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid if self.fetch_values_from: - from_dttm = utils.parse_human_datetime(self.fetch_values_from) + from_dttm = parse_human_datetime(self.fetch_values_from) assert from_dttm else: from_dttm = datetime(1970, 1, 1) @@ -1426,7 +1427,7 @@ class DruidDatasource(Model, BaseDatasource): time_offset = DruidDatasource.time_offset(query_obj["granularity"]) def increment_timestamp(ts: str) -> datetime: - dt = utils.parse_human_datetime(ts).replace(tzinfo=DRUID_TZ) + dt = parse_human_datetime(ts).replace(tzinfo=DRUID_TZ) return dt + timedelta(milliseconds=time_offset) if DTTM_ALIAS in df.columns and time_offset: diff --git a/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py b/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py index ca4de4e3e2..1d0d81faaf 100644 --- a/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py +++ b/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py @@ -33,7 +33,7 @@ from sqlalchemy import Column, Integer, String, Text from sqlalchemy.ext.declarative import declarative_base from superset import db -from superset.utils.core import parse_human_timedelta +from superset.utils.date_parser import parse_human_timedelta revision = "3dda56f1c4c6" down_revision = "bddc498dd179" diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py index 953ed3154d..e32467d0c3 100644 --- a/superset/tasks/cache.py +++ b/superset/tasks/cache.py @@ -31,7 +31,7 @@ from superset.models.core import Log from superset.models.dashboard import Dashboard from superset.models.slice import Slice from superset.models.tags import Tag, TaggedObject -from superset.utils.core import parse_human_datetime +from superset.utils.date_parser import parse_human_datetime from superset.views.utils import build_extra_filters logger = get_task_logger(__name__) diff --git a/superset/utils/core.py b/superset/utils/core.py index c500c197ac..7219317ee9 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. """Utility functions used across Superset""" -import calendar import decimal import errno import functools @@ -39,7 +38,6 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate from enum import Enum -from time import struct_time from timeit import default_timer from types import TracebackType from typing import ( @@ -65,29 +63,14 @@ import bleach import markdown as md import numpy as np import pandas as pd -import parsedatetime import sqlalchemy as sa from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends.openssl.x509 import _Certificate -from dateutil.parser import parse -from dateutil.relativedelta import relativedelta from flask import current_app, flash, g, Markup, render_template from flask_appbuilder import SQLA from flask_appbuilder.security.sqla.models import Role, User -from flask_babel import gettext as __, lazy_gettext as _ -from holidays import CountryHoliday -from pyparsing import ( - CaselessKeyword, - Forward, - Group, - Optional as ppOptional, - ParseException, - ParseResults, - pyparsing_common, - quotedString, - Suppress, -) +from flask_babel import gettext as __ from sqlalchemy import event, exc, select, Text from sqlalchemy.dialects.mysql import MEDIUMTEXT from sqlalchemy.engine import Connection, Engine @@ -443,58 +426,6 @@ def list_minus(l: List[Any], minus: List[Any]) -> List[Any]: return [o for o in l if o not in minus] -def parse_human_datetime(human_readable: str) -> datetime: - """ - Returns ``datetime.datetime`` from human readable strings - - >>> from datetime import date, timedelta - >>> from dateutil.relativedelta import relativedelta - >>> parse_human_datetime('2015-04-03') - datetime.datetime(2015, 4, 3, 0, 0) - >>> parse_human_datetime('2/3/1969') - datetime.datetime(1969, 2, 3, 0, 0) - >>> parse_human_datetime('now') <= datetime.now() - True - >>> parse_human_datetime('yesterday') <= datetime.now() - True - >>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date() - True - >>> year_ago_1 = parse_human_datetime('one year ago').date() - >>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date() - >>> year_ago_1 == year_ago_2 - True - >>> year_after_1 = parse_human_datetime('2 years after').date() - >>> year_after_2 = (datetime.now() + relativedelta(years=2)).date() - >>> year_after_1 == year_after_2 - True - """ - try: - dttm = parse(human_readable) - except Exception: # pylint: disable=broad-except - try: - cal = parsedatetime.Calendar() - parsed_dttm, parsed_flags = cal.parseDT(human_readable) - # when time is not extracted, we 'reset to midnight' - if parsed_flags & 2 == 0: - parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0) - dttm = dttm_from_timetuple(parsed_dttm.utctimetuple()) - except Exception as ex: - logger.exception(ex) - raise ValueError("Couldn't parse date string [{}]".format(human_readable)) - return dttm - - -def dttm_from_timetuple(date_: struct_time) -> datetime: - return datetime( - date_.tm_year, - date_.tm_mon, - date_.tm_mday, - date_.tm_hour, - date_.tm_min, - date_.tm_sec, - ) - - def md5_hex(data: str) -> str: return hashlib.md5(data.encode()).hexdigest() @@ -516,39 +447,6 @@ class DashboardEncoder(json.JSONEncoder): return json.JSONEncoder(sort_keys=True).default(o) -def parse_human_timedelta( - human_readable: Optional[str], source_time: Optional[datetime] = None, -) -> timedelta: - """ - Returns ``datetime.timedelta`` from natural language time deltas - - >>> parse_human_timedelta('1 day') == timedelta(days=1) - True - """ - cal = parsedatetime.Calendar() - source_dttm = dttm_from_timetuple( - source_time.timetuple() if source_time else datetime.now().timetuple() - ) - modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0]) - return modified_dttm - source_dttm - - -def parse_past_timedelta( - delta_str: str, source_time: Optional[datetime] = None -) -> timedelta: - """ - Takes a delta like '1 year' and finds the timedelta for that period in - the past, then represents that past timedelta in positive terms. - - parse_human_timedelta('1 year') find the timedelta 1 year in the future. - parse_past_timedelta('1 year') returns -datetime.timedelta(-365) - or datetime.timedelta(365). - """ - return -parse_human_timedelta( - delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time, - ) - - class JSONEncodedDict(TypeDecorator): # pylint: disable=abstract-method """Represents an immutable structure as a json-encoded string.""" @@ -1254,347 +1152,6 @@ def ensure_path_exists(path: str) -> None: raise -class EvalText: # pylint: disable=too-few-public-methods - def __init__(self, tokens: ParseResults) -> None: - self.value = tokens[0] - - def eval(self) -> str: - # strip quotes - return self.value[1:-1] - - -class EvalDateTimeFunc: # pylint: disable=too-few-public-methods - def __init__(self, tokens: ParseResults) -> None: - self.value = tokens[1] - - def eval(self) -> datetime: - return parse_human_datetime(self.value.eval()) - - -class EvalDateAddFunc: # pylint: disable=too-few-public-methods - def __init__(self, tokens: ParseResults) -> None: - self.value = tokens[1] - - def eval(self) -> datetime: - dttm_expression, delta, unit = self.value - dttm = dttm_expression.eval() - if unit.lower() == "quarter": - delta = delta * 3 - unit = "month" - return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm) - - -class EvalDateTruncFunc: # pylint: disable=too-few-public-methods - def __init__(self, tokens: ParseResults) -> None: - self.value = tokens[1] - - def eval(self) -> datetime: - dttm_expression, unit = self.value - dttm = dttm_expression.eval() - if unit == "year": - dttm = dttm.replace( - month=1, day=1, hour=0, minute=0, second=0, microsecond=0 - ) - elif unit == "month": - dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - elif unit == "week": - dttm = dttm - relativedelta(days=dttm.weekday()) - dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0) - elif unit == "day": - dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0) - elif unit == "hour": - dttm = dttm.replace(minute=0, second=0, microsecond=0) - elif unit == "minute": - dttm = dttm.replace(second=0, microsecond=0) - else: - dttm = dttm.replace(microsecond=0) - return dttm - - -class EvalLastDayFunc: # pylint: disable=too-few-public-methods - def __init__(self, tokens: ParseResults) -> None: - self.value = tokens[1] - - def eval(self) -> datetime: - dttm_expression, unit = self.value - dttm = dttm_expression.eval() - if unit == "year": - return dttm.replace( - month=12, day=31, hour=0, minute=0, second=0, microsecond=0 - ) - if unit == "month": - return dttm.replace( - day=calendar.monthrange(dttm.year, dttm.month)[1], - hour=0, - minute=0, - second=0, - microsecond=0, - ) - # unit == "week": - mon = dttm - relativedelta(days=dttm.weekday()) - mon = mon.replace(hour=0, minute=0, second=0, microsecond=0) - return mon + relativedelta(days=6) - - -class EvalHolidayFunc: # pylint: disable=too-few-public-methods - def __init__(self, tokens: ParseResults) -> None: - self.value = tokens[1] - - def eval(self) -> datetime: - holiday = self.value[0].eval() - dttm, country = [None, None] - if len(self.value) >= 2: - dttm = self.value[1].eval() - if len(self.value) == 3: - country = self.value[2] - holiday_year = dttm.year if dttm else parse_human_datetime("today").year - country = country.eval() if country else "US" - - holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False) - searched_result = holiday_lookup.get_named(holiday) - if len(searched_result) == 1: - return dttm_from_timetuple(searched_result[0].timetuple()) - raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday)) - - -@memoized() -def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals - ( # pylint: disable=invalid-name - DATETIME, - DATEADD, - DATETRUNC, - LASTDAY, - HOLIDAY, - YEAR, - QUARTER, - MONTH, - WEEK, - DAY, - HOUR, - MINUTE, - SECOND, - ) = map( - CaselessKeyword, - "datetime dateadd datetrunc lastday holiday " - "year quarter month week day hour minute second".split(), - ) - lparen, rparen, comma = map(Suppress, "(),") - int_operand = pyparsing_common.signed_integer().setName("int_operand") - text_operand = quotedString.setName("text_operand").setParseAction(EvalText) - - # allow expression to be used recursively - datetime_func = Forward().setName("datetime") - dateadd_func = Forward().setName("dateadd") - datetrunc_func = Forward().setName("datetrunc") - lastday_func = Forward().setName("lastday") - holiday_func = Forward().setName("holiday") - date_expr = ( - datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func - ) - - datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction( - EvalDateTimeFunc - ) - dateadd_func <<= ( - DATEADD - + lparen - + Group( - date_expr - + comma - + int_operand - + comma - + (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND) - + ppOptional(comma) - ) - + rparen - ).setParseAction(EvalDateAddFunc) - datetrunc_func <<= ( - DATETRUNC - + lparen - + Group( - date_expr - + comma - + (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND) - + ppOptional(comma) - ) - + rparen - ).setParseAction(EvalDateTruncFunc) - lastday_func <<= ( - LASTDAY - + lparen - + Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma)) - + rparen - ).setParseAction(EvalLastDayFunc) - holiday_func <<= ( - HOLIDAY - + lparen - + Group( - text_operand - + ppOptional(comma) - + ppOptional(date_expr) - + ppOptional(comma) - + ppOptional(text_operand) - + ppOptional(comma) - ) - + rparen - ).setParseAction(EvalHolidayFunc) - - return date_expr - - -def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]: - if datetime_expression: - try: - return datetime_parser().parseString(datetime_expression)[0].eval() - except ParseException as error: - raise ValueError(error) - return None - - -# pylint: disable=too-many-arguments, too-many-locals, too-many-branches -def get_since_until( - time_range: Optional[str] = None, - since: Optional[str] = None, - until: Optional[str] = None, - time_shift: Optional[str] = None, - relative_start: Optional[str] = None, - relative_end: Optional[str] = None, -) -> Tuple[Optional[datetime], Optional[datetime]]: - """Return `since` and `until` date time tuple from string representations of - time_range, since, until and time_shift. - - This functiom supports both reading the keys separately (from `since` and - `until`), as well as the new `time_range` key. Valid formats are: - - - ISO 8601 - - X days/years/hours/day/year/weeks - - X days/years/hours/day/year/weeks ago - - X days/years/hours/day/year/weeks from now - - freeform - - Additionally, for `time_range` (these specify both `since` and `until`): - - - Last day - - Last week - - Last month - - Last quarter - - Last year - - No filter - - Last X seconds/minutes/hours/days/weeks/months/years - - Next X seconds/minutes/hours/days/weeks/months/years - - """ - separator = " : " - _relative_start = relative_start if relative_start else "today" - _relative_end = relative_end if relative_end else "today" - - if time_range == "No filter": - return None, None - - if time_range and time_range.startswith("Last") and separator not in time_range: - time_range = time_range + separator + _relative_end - - if time_range and time_range.startswith("Next") and separator not in time_range: - time_range = _relative_start + separator + time_range - - if ( - time_range - and time_range.startswith("previous calendar week") - and separator not in time_range - ): - time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long - if ( - time_range - and time_range.startswith("previous calendar month") - and separator not in time_range - ): - time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long - if ( - time_range - and time_range.startswith("previous calendar year") - and separator not in time_range - ): - time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long - - if time_range and separator in time_range: - time_range_lookup = [ - ( - r"^last\s+(day|week|month|quarter|year)$", - lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})", - ), - ( - r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$", - lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long - ), - ( - r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$", - lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long - ), - ( - r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$", - lambda text: text, - ), - ] - - since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)] - since_and_until: List[Optional[str]] = [] - for part in since_and_until_partition: - if not part: - # if since or until is "", set as None - since_and_until.append(None) - continue - - # Is it possible to match to time_range_lookup - matched = False - for pattern, fn in time_range_lookup: - result = re.search(pattern, part, re.IGNORECASE) - if result: - matched = True - # converted matched time_range to "formal time expressions" - since_and_until.append(fn(*result.groups())) # type: ignore - if not matched: - # default matched case - since_and_until.append(f"DATETIME('{part}')") - - _since, _until = map(datetime_eval, since_and_until) - else: - since = since or "" - if since: - since = add_ago_to_since(since) - _since = parse_human_datetime(since) if since else None - _until = ( - parse_human_datetime(until) - if until - else parse_human_datetime(_relative_end) - ) - - if time_shift: - time_delta = parse_past_timedelta(time_shift) - _since = _since if _since is None else (_since - time_delta) - _until = _until if _until is None else (_until - time_delta) - - if _since and _until and _since > _until: - raise ValueError(_("From date cannot be larger than to date")) - - return _since, _until - - -def add_ago_to_since(since: str) -> str: - """ - Backwards compatibility hack. Without this slices with since: 7 days will - be treated as 7 days in the future. - - :param str since: - :returns: Since with ago added if necessary - :rtype: str - """ - since_words = since.split(" ") - grains = ["days", "years", "hours", "day", "year", "weeks"] - if len(since_words) == 2 and since_words[1] in grains: - since += " ago" - return since - - def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name form_data: FormData, ) -> None: diff --git a/superset/utils/date_parser.py b/superset/utils/date_parser.py new file mode 100644 index 0000000000..aee2c83a0d --- /dev/null +++ b/superset/utils/date_parser.py @@ -0,0 +1,469 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import calendar +import logging +import re +from datetime import datetime, timedelta +from time import struct_time +from typing import List, Optional, Tuple + +import parsedatetime +from dateutil.parser import parse +from dateutil.relativedelta import relativedelta +from flask_babel import lazy_gettext as _ +from holidays import CountryHoliday +from pyparsing import ( + CaselessKeyword, + Forward, + Group, + Optional as ppOptional, + ParseException, + ParseResults, + pyparsing_common, + quotedString, + Suppress, +) + +from .core import memoized + +logger = logging.getLogger(__name__) + + +def parse_human_datetime(human_readable: str) -> datetime: + """ + Returns ``datetime.datetime`` from human readable strings + + >>> from datetime import date, timedelta + >>> from dateutil.relativedelta import relativedelta + >>> parse_human_datetime('2015-04-03') + datetime.datetime(2015, 4, 3, 0, 0) + >>> parse_human_datetime('2/3/1969') + datetime.datetime(1969, 2, 3, 0, 0) + >>> parse_human_datetime('now') <= datetime.now() + True + >>> parse_human_datetime('yesterday') <= datetime.now() + True + >>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date() + True + >>> year_ago_1 = parse_human_datetime('one year ago').date() + >>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date() + >>> year_ago_1 == year_ago_2 + True + >>> year_after_1 = parse_human_datetime('2 years after').date() + >>> year_after_2 = (datetime.now() + relativedelta(years=2)).date() + >>> year_after_1 == year_after_2 + True + """ + try: + dttm = parse(human_readable) + except Exception: # pylint: disable=broad-except + try: + cal = parsedatetime.Calendar() + parsed_dttm, parsed_flags = cal.parseDT(human_readable) + # when time is not extracted, we 'reset to midnight' + if parsed_flags & 2 == 0: + parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0) + dttm = dttm_from_timetuple(parsed_dttm.utctimetuple()) + except Exception as ex: + logger.exception(ex) + raise ValueError("Couldn't parse date string [{}]".format(human_readable)) + return dttm + + +def dttm_from_timetuple(date_: struct_time) -> datetime: + return datetime( + date_.tm_year, + date_.tm_mon, + date_.tm_mday, + date_.tm_hour, + date_.tm_min, + date_.tm_sec, + ) + + +def parse_human_timedelta( + human_readable: Optional[str], source_time: Optional[datetime] = None, +) -> timedelta: + """ + Returns ``datetime.timedelta`` from natural language time deltas + + >>> parse_human_timedelta('1 day') == timedelta(days=1) + True + """ + cal = parsedatetime.Calendar() + source_dttm = dttm_from_timetuple( + source_time.timetuple() if source_time else datetime.now().timetuple() + ) + modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0]) + return modified_dttm - source_dttm + + +def parse_past_timedelta( + delta_str: str, source_time: Optional[datetime] = None +) -> timedelta: + """ + Takes a delta like '1 year' and finds the timedelta for that period in + the past, then represents that past timedelta in positive terms. + + parse_human_timedelta('1 year') find the timedelta 1 year in the future. + parse_past_timedelta('1 year') returns -datetime.timedelta(-365) + or datetime.timedelta(365). + """ + return -parse_human_timedelta( + delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time, + ) + + +# pylint: disable=too-many-arguments, too-many-locals, too-many-branches +def get_since_until( + time_range: Optional[str] = None, + since: Optional[str] = None, + until: Optional[str] = None, + time_shift: Optional[str] = None, + relative_start: Optional[str] = None, + relative_end: Optional[str] = None, +) -> Tuple[Optional[datetime], Optional[datetime]]: + """Return `since` and `until` date time tuple from string representations of + time_range, since, until and time_shift. + + This functiom supports both reading the keys separately (from `since` and + `until`), as well as the new `time_range` key. Valid formats are: + + - ISO 8601 + - X days/years/hours/day/year/weeks + - X days/years/hours/day/year/weeks ago + - X days/years/hours/day/year/weeks from now + - freeform + + Additionally, for `time_range` (these specify both `since` and `until`): + + - Last day + - Last week + - Last month + - Last quarter + - Last year + - No filter + - Last X seconds/minutes/hours/days/weeks/months/years + - Next X seconds/minutes/hours/days/weeks/months/years + + """ + separator = " : " + _relative_start = relative_start if relative_start else "today" + _relative_end = relative_end if relative_end else "today" + + if time_range == "No filter": + return None, None + + if time_range and time_range.startswith("Last") and separator not in time_range: + time_range = time_range + separator + _relative_end + + if time_range and time_range.startswith("Next") and separator not in time_range: + time_range = _relative_start + separator + time_range + + if ( + time_range + and time_range.startswith("previous calendar week") + and separator not in time_range + ): + time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long + if ( + time_range + and time_range.startswith("previous calendar month") + and separator not in time_range + ): + time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long + if ( + time_range + and time_range.startswith("previous calendar year") + and separator not in time_range + ): + time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long + + if time_range and separator in time_range: + time_range_lookup = [ + ( + r"^last\s+(day|week|month|quarter|year)$", + lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})", + ), + ( + r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$", + lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long + ), + ( + r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$", + lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long + ), + ( + r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$", + lambda text: text, + ), + ] + + since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)] + since_and_until: List[Optional[str]] = [] + for part in since_and_until_partition: + if not part: + # if since or until is "", set as None + since_and_until.append(None) + continue + + # Is it possible to match to time_range_lookup + matched = False + for pattern, fn in time_range_lookup: + result = re.search(pattern, part, re.IGNORECASE) + if result: + matched = True + # converted matched time_range to "formal time expressions" + since_and_until.append(fn(*result.groups())) # type: ignore + if not matched: + # default matched case + since_and_until.append(f"DATETIME('{part}')") + + _since, _until = map(datetime_eval, since_and_until) + else: + since = since or "" + if since: + since = add_ago_to_since(since) + _since = parse_human_datetime(since) if since else None + _until = ( + parse_human_datetime(until) + if until + else parse_human_datetime(_relative_end) + ) + + if time_shift: + time_delta = parse_past_timedelta(time_shift) + _since = _since if _since is None else (_since - time_delta) + _until = _until if _until is None else (_until - time_delta) + + if _since and _until and _since > _until: + raise ValueError(_("From date cannot be larger than to date")) + + return _since, _until + + +def add_ago_to_since(since: str) -> str: + """ + Backwards compatibility hack. Without this slices with since: 7 days will + be treated as 7 days in the future. + + :param str since: + :returns: Since with ago added if necessary + :rtype: str + """ + since_words = since.split(" ") + grains = ["days", "years", "hours", "day", "year", "weeks"] + if len(since_words) == 2 and since_words[1] in grains: + since += " ago" + return since + + +class EvalText: # pylint: disable=too-few-public-methods + def __init__(self, tokens: ParseResults) -> None: + self.value = tokens[0] + + def eval(self) -> str: + # strip quotes + return self.value[1:-1] + + +class EvalDateTimeFunc: # pylint: disable=too-few-public-methods + def __init__(self, tokens: ParseResults) -> None: + self.value = tokens[1] + + def eval(self) -> datetime: + return parse_human_datetime(self.value.eval()) + + +class EvalDateAddFunc: # pylint: disable=too-few-public-methods + def __init__(self, tokens: ParseResults) -> None: + self.value = tokens[1] + + def eval(self) -> datetime: + dttm_expression, delta, unit = self.value + dttm = dttm_expression.eval() + if unit.lower() == "quarter": + delta = delta * 3 + unit = "month" + return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm) + + +class EvalDateTruncFunc: # pylint: disable=too-few-public-methods + def __init__(self, tokens: ParseResults) -> None: + self.value = tokens[1] + + def eval(self) -> datetime: + dttm_expression, unit = self.value + dttm = dttm_expression.eval() + if unit == "year": + dttm = dttm.replace( + month=1, day=1, hour=0, minute=0, second=0, microsecond=0 + ) + elif unit == "month": + dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + elif unit == "week": + dttm = dttm - relativedelta(days=dttm.weekday()) + dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0) + elif unit == "day": + dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0) + elif unit == "hour": + dttm = dttm.replace(minute=0, second=0, microsecond=0) + elif unit == "minute": + dttm = dttm.replace(second=0, microsecond=0) + else: + dttm = dttm.replace(microsecond=0) + return dttm + + +class EvalLastDayFunc: # pylint: disable=too-few-public-methods + def __init__(self, tokens: ParseResults) -> None: + self.value = tokens[1] + + def eval(self) -> datetime: + dttm_expression, unit = self.value + dttm = dttm_expression.eval() + if unit == "year": + return dttm.replace( + month=12, day=31, hour=0, minute=0, second=0, microsecond=0 + ) + if unit == "month": + return dttm.replace( + day=calendar.monthrange(dttm.year, dttm.month)[1], + hour=0, + minute=0, + second=0, + microsecond=0, + ) + # unit == "week": + mon = dttm - relativedelta(days=dttm.weekday()) + mon = mon.replace(hour=0, minute=0, second=0, microsecond=0) + return mon + relativedelta(days=6) + + +class EvalHolidayFunc: # pylint: disable=too-few-public-methods + def __init__(self, tokens: ParseResults) -> None: + self.value = tokens[1] + + def eval(self) -> datetime: + holiday = self.value[0].eval() + dttm, country = [None, None] + if len(self.value) >= 2: + dttm = self.value[1].eval() + if len(self.value) == 3: + country = self.value[2] + holiday_year = dttm.year if dttm else parse_human_datetime("today").year + country = country.eval() if country else "US" + + holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False) + searched_result = holiday_lookup.get_named(holiday) + if len(searched_result) == 1: + return dttm_from_timetuple(searched_result[0].timetuple()) + raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday)) + + +@memoized() +def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals + ( # pylint: disable=invalid-name + DATETIME, + DATEADD, + DATETRUNC, + LASTDAY, + HOLIDAY, + YEAR, + QUARTER, + MONTH, + WEEK, + DAY, + HOUR, + MINUTE, + SECOND, + ) = map( + CaselessKeyword, + "datetime dateadd datetrunc lastday holiday " + "year quarter month week day hour minute second".split(), + ) + lparen, rparen, comma = map(Suppress, "(),") + int_operand = pyparsing_common.signed_integer().setName("int_operand") + text_operand = quotedString.setName("text_operand").setParseAction(EvalText) + + # allow expression to be used recursively + datetime_func = Forward().setName("datetime") + dateadd_func = Forward().setName("dateadd") + datetrunc_func = Forward().setName("datetrunc") + lastday_func = Forward().setName("lastday") + holiday_func = Forward().setName("holiday") + date_expr = ( + datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func + ) + + datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction( + EvalDateTimeFunc + ) + dateadd_func <<= ( + DATEADD + + lparen + + Group( + date_expr + + comma + + int_operand + + comma + + (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND) + + ppOptional(comma) + ) + + rparen + ).setParseAction(EvalDateAddFunc) + datetrunc_func <<= ( + DATETRUNC + + lparen + + Group( + date_expr + + comma + + (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND) + + ppOptional(comma) + ) + + rparen + ).setParseAction(EvalDateTruncFunc) + lastday_func <<= ( + LASTDAY + + lparen + + Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma)) + + rparen + ).setParseAction(EvalLastDayFunc) + holiday_func <<= ( + HOLIDAY + + lparen + + Group( + text_operand + + ppOptional(comma) + + ppOptional(date_expr) + + ppOptional(comma) + + ppOptional(text_operand) + + ppOptional(comma) + ) + + rparen + ).setParseAction(EvalHolidayFunc) + + return date_expr + + +def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]: + if datetime_expression: + try: + return datetime_parser().parseString(datetime_expression)[0].eval() + except ParseException as error: + raise ValueError(error) + return None diff --git a/superset/views/api.py b/superset/views/api.py index de7d656cb5..7df1c5e71b 100644 --- a/superset/views/api.py +++ b/superset/views/api.py @@ -29,7 +29,7 @@ from superset.legacy import update_time_range from superset.models.slice import Slice from superset.typing import FlaskResponse from superset.utils import core as utils -from superset.utils.core import get_since_until +from superset.utils.date_parser import get_since_until from superset.views.base import api, BaseSupersetView, handle_api_exception get_time_range_schema = {"type": "string"} diff --git a/superset/viz.py b/superset/viz.py index 6baef6a046..21fbdf810c 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -75,6 +75,7 @@ from superset.utils.core import ( QueryMode, to_adhoc, ) +from superset.utils.date_parser import get_since_until, parse_past_timedelta from superset.utils.dates import datetime_to_epoch from superset.utils.hashing import md5_sha_from_str @@ -356,7 +357,7 @@ class BaseViz: order_desc = form_data.get("order_desc", True) try: - since, until = utils.get_since_until( + since, until = get_since_until( relative_start=relative_start, relative_end=relative_end, time_range=form_data.get("time_range"), @@ -367,7 +368,7 @@ class BaseViz: raise QueryObjectValidationError(str(ex)) time_shift = form_data.get("time_shift", "") - self.time_shift = utils.parse_past_timedelta(time_shift) + self.time_shift = parse_past_timedelta(time_shift) from_dttm = None if since is None else (since - self.time_shift) to_dttm = None if until is None else (until - self.time_shift) if from_dttm and to_dttm and from_dttm > to_dttm: @@ -1004,7 +1005,7 @@ class CalHeatmapViz(BaseViz): data[metric] = values try: - start, end = utils.get_since_until( + start, end = get_since_until( relative_start=relative_start, relative_end=relative_end, time_range=form_data.get("time_range"), @@ -1318,7 +1319,7 @@ class NVD3TimeSeriesViz(NVD3Viz): for option in time_compare: query_object = self.query_obj() try: - delta = utils.parse_past_timedelta(option) + delta = parse_past_timedelta(option) except ValueError as ex: raise QueryObjectValidationError(str(ex)) query_object["inner_from_dttm"] = query_object["from_dttm"] diff --git a/superset/viz_sip38.py b/superset/viz_sip38.py index 5a16639975..9ec1752741 100644 --- a/superset/viz_sip38.py +++ b/superset/viz_sip38.py @@ -63,6 +63,7 @@ from superset.utils.core import ( merge_extra_filters, to_adhoc, ) +from superset.utils.date_parser import get_since_until, parse_past_timedelta import dataclasses # isort:skip @@ -359,7 +360,7 @@ class BaseViz: # default order direction order_desc = form_data.get("order_desc", True) - since, until = utils.get_since_until( + since, until = get_since_until( relative_start=relative_start, relative_end=relative_end, time_range=form_data.get("time_range"), @@ -367,7 +368,7 @@ class BaseViz: until=form_data.get("until"), ) time_shift = form_data.get("time_shift", "") - self.time_shift = utils.parse_past_timedelta(time_shift) + self.time_shift = parse_past_timedelta(time_shift) from_dttm = None if since is None else (since - self.time_shift) to_dttm = None if until is None else (until - self.time_shift) if from_dttm and to_dttm and from_dttm > to_dttm: @@ -883,7 +884,7 @@ class CalHeatmapViz(BaseViz): values[str(v / 10 ** 9)] = obj.get(metric) data[metric] = values - start, end = utils.get_since_until( + start, end = get_since_until( relative_start=relative_start, relative_end=relative_end, time_range=form_data.get("time_range"), @@ -1265,7 +1266,7 @@ class NVD3TimeSeriesViz(NVD3Viz): for option in time_compare: query_object = self.query_obj() - delta = utils.parse_past_timedelta(option) + delta = parse_past_timedelta(option) query_object["inner_from_dttm"] = query_object["from_dttm"] query_object["inner_to_dttm"] = query_object["to_dttm"] diff --git a/tests/utils/date_parser_tests.py b/tests/utils/date_parser_tests.py new file mode 100644 index 0000000000..fc592d3ee6 --- /dev/null +++ b/tests/utils/date_parser_tests.py @@ -0,0 +1,263 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime, timedelta +from unittest.mock import patch + +from superset.utils.date_parser import ( + datetime_eval, + get_since_until, + parse_human_timedelta, + parse_past_timedelta, +) +from tests.base_tests import SupersetTestCase + + +def mock_parse_human_datetime(s): + if s == "now": + return datetime(2016, 11, 7, 9, 30, 10) + elif s == "today": + return datetime(2016, 11, 7) + elif s == "yesterday": + return datetime(2016, 11, 6) + elif s == "tomorrow": + return datetime(2016, 11, 8) + elif s == "Last year": + return datetime(2015, 11, 7) + elif s == "Last week": + return datetime(2015, 10, 31) + elif s == "Last 5 months": + return datetime(2016, 6, 7) + elif s == "Next 5 months": + return datetime(2017, 4, 7) + elif s in ["5 days", "5 days ago"]: + return datetime(2016, 11, 2) + elif s == "2018-01-01T00:00:00": + return datetime(2018, 1, 1) + elif s == "2018-12-31T23:59:59": + return datetime(2018, 12, 31, 23, 59, 59) + + +class TestDateParser(SupersetTestCase): + @patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime) + def test_get_since_until(self): + result = get_since_until() + expected = None, datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until(" : now") + expected = None, datetime(2016, 11, 7, 9, 30, 10) + self.assertEqual(result, expected) + + result = get_since_until("yesterday : tomorrow") + expected = datetime(2016, 11, 6), datetime(2016, 11, 8) + self.assertEqual(result, expected) + + result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59") + expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59) + self.assertEqual(result, expected) + + result = get_since_until("Last year") + expected = datetime(2015, 11, 7), datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until("Last quarter") + expected = datetime(2016, 8, 7), datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until("Last 5 months") + expected = datetime(2016, 6, 7), datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until("Next 5 months") + expected = datetime(2016, 11, 7), datetime(2017, 4, 7) + self.assertEqual(result, expected) + + result = get_since_until(since="5 days") + expected = datetime(2016, 11, 2), datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until(since="5 days ago", until="tomorrow") + expected = datetime(2016, 11, 2), datetime(2016, 11, 8) + self.assertEqual(result, expected) + + result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day") + expected = datetime(2016, 11, 5), datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until(time_range="5 days : now") + expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10) + self.assertEqual(result, expected) + + result = get_since_until("Last week", relative_end="now") + expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10) + self.assertEqual(result, expected) + + result = get_since_until("Last week", relative_start="now") + expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7) + self.assertEqual(result, expected) + + result = get_since_until("Last week", relative_start="now", relative_end="now") + expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10) + self.assertEqual(result, expected) + + result = get_since_until("previous calendar week") + expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0) + self.assertEqual(result, expected) + + result = get_since_until("previous calendar month") + expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0) + self.assertEqual(result, expected) + + result = get_since_until("previous calendar year") + expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0) + self.assertEqual(result, expected) + + with self.assertRaises(ValueError): + get_since_until(time_range="tomorrow : yesterday") + + @patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime) + def test_datetime_eval(self): + result = datetime_eval("datetime('now')") + expected = datetime(2016, 11, 7, 9, 30, 10) + self.assertEqual(result, expected) + + result = datetime_eval("datetime('today' )") + expected = datetime(2016, 11, 7) + self.assertEqual(result, expected) + + # Parse compact arguments spelling + result = datetime_eval("dateadd(datetime('today'),1,year,)") + expected = datetime(2017, 11, 7) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('today'), -2, year)") + expected = datetime(2014, 11, 7) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('today'), 2, quarter)") + expected = datetime(2017, 5, 7) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('today'), 3, month)") + expected = datetime(2017, 2, 7) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('today'), -3, week)") + expected = datetime(2016, 10, 17) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('today'), 3, day)") + expected = datetime(2016, 11, 10) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('now'), 3, hour)") + expected = datetime(2016, 11, 7, 12, 30, 10) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('now'), 40, minute)") + expected = datetime(2016, 11, 7, 10, 10, 10) + self.assertEqual(result, expected) + + result = datetime_eval("dateadd(datetime('now'), -11, second)") + expected = datetime(2016, 11, 7, 9, 29, 59) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), year)") + expected = datetime(2016, 1, 1, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), month)") + expected = datetime(2016, 11, 1, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), day)") + expected = datetime(2016, 11, 7, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), week)") + expected = datetime(2016, 11, 7, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), hour)") + expected = datetime(2016, 11, 7, 9, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), minute)") + expected = datetime(2016, 11, 7, 9, 30, 0) + self.assertEqual(result, expected) + + result = datetime_eval("datetrunc(datetime('now'), second)") + expected = datetime(2016, 11, 7, 9, 30, 10) + self.assertEqual(result, expected) + + result = datetime_eval("lastday(datetime('now'), year)") + expected = datetime(2016, 12, 31, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("lastday(datetime('today'), month)") + expected = datetime(2016, 11, 30, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("holiday('Christmas')") + expected = datetime(2016, 12, 25, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))") + expected = datetime(2018, 9, 3, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval( + "holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')" + ) + expected = datetime(2018, 12, 26, 0, 0, 0) + self.assertEqual(result, expected) + + result = datetime_eval( + "lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)" + ) + expected = datetime(2018, 2, 28, 0, 0, 0) + self.assertEqual(result, expected) + + @patch("superset.utils.date_parser.datetime") + def test_parse_human_timedelta(self, mock_datetime): + mock_datetime.now.return_value = datetime(2019, 4, 1) + mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw) + self.assertEqual(parse_human_timedelta("now"), timedelta(0)) + self.assertEqual(parse_human_timedelta("1 year"), timedelta(366)) + self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365)) + self.assertEqual(parse_human_timedelta(None), timedelta(0)) + self.assertEqual( + parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30), + ) + self.assertEqual( + parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31), + ) + self.assertEqual( + parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28), + ) + self.assertEqual( + parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31), + ) + + @patch("superset.utils.date_parser.datetime") + def test_parse_past_timedelta(self, mock_datetime): + mock_datetime.now.return_value = datetime(2019, 4, 1) + mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw) + self.assertEqual(parse_past_timedelta("1 year"), timedelta(365)) + self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365)) + self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364)) + self.assertEqual(parse_past_timedelta("1 month"), timedelta(31)) diff --git a/tests/utils_tests.py b/tests/utils_tests.py index 293c2d49c4..a16db4b27b 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -48,7 +48,6 @@ from superset.utils.core import ( get_iterable, get_email_address_list, get_or_create_db, - get_since_until, get_stacktrace, json_int_dttm_ser, json_iso_dttm_ser, @@ -57,15 +56,12 @@ from superset.utils.core import ( merge_extra_filters, merge_request_params, parse_ssl_cert, - parse_human_timedelta, parse_js_uri_path_item, - parse_past_timedelta, split, TimeRangeEndpoint, validate_json, zlib_compress, zlib_decompress, - datetime_eval, ) from superset.utils import schema from superset.views.utils import ( @@ -78,31 +74,6 @@ from tests.base_tests import SupersetTestCase from .fixtures.certificates import ssl_certificate -def mock_parse_human_datetime(s): - if s == "now": - return datetime(2016, 11, 7, 9, 30, 10) - elif s == "today": - return datetime(2016, 11, 7) - elif s == "yesterday": - return datetime(2016, 11, 6) - elif s == "tomorrow": - return datetime(2016, 11, 8) - elif s == "Last year": - return datetime(2015, 11, 7) - elif s == "Last week": - return datetime(2015, 10, 31) - elif s == "Last 5 months": - return datetime(2016, 6, 7) - elif s == "Next 5 months": - return datetime(2017, 4, 7) - elif s in ["5 days", "5 days ago"]: - return datetime(2016, 11, 2) - elif s == "2018-01-01T00:00:00": - return datetime(2018, 1, 1) - elif s == "2018-12-31T23:59:59": - return datetime(2018, 12, 31, 23, 59, 59) - - def mock_to_adhoc(filt, expressionType="SIMPLE", clause="where"): result = {"clause": clause.upper(), "expressionType": expressionType} @@ -149,36 +120,6 @@ class TestUtils(SupersetTestCase): assert isinstance(base_json_conv(uuid.uuid4()), str) is True assert isinstance(base_json_conv(timedelta(0)), str) is True - @patch("superset.utils.core.datetime") - def test_parse_human_timedelta(self, mock_datetime): - mock_datetime.now.return_value = datetime(2019, 4, 1) - mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw) - self.assertEqual(parse_human_timedelta("now"), timedelta(0)) - self.assertEqual(parse_human_timedelta("1 year"), timedelta(366)) - self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365)) - self.assertEqual(parse_human_timedelta(None), timedelta(0)) - self.assertEqual( - parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30), - ) - self.assertEqual( - parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31), - ) - self.assertEqual( - parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28), - ) - self.assertEqual( - parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31), - ) - - @patch("superset.utils.core.datetime") - def test_parse_past_timedelta(self, mock_datetime): - mock_datetime.now.return_value = datetime(2019, 4, 1) - mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw) - self.assertEqual(parse_past_timedelta("1 year"), timedelta(365)) - self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365)) - self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364)) - self.assertEqual(parse_past_timedelta("1 month"), timedelta(31)) - def test_zlib_compression(self): json_str = '{"test": 1}' blob = zlib_compress(json_str) @@ -701,186 +642,6 @@ class TestUtils(SupersetTestCase): self.assertEqual(instance.watcher, 4) self.assertEqual(result1, result8) - @patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime) - def test_get_since_until(self): - result = get_since_until() - expected = None, datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until(" : now") - expected = None, datetime(2016, 11, 7, 9, 30, 10) - self.assertEqual(result, expected) - - result = get_since_until("yesterday : tomorrow") - expected = datetime(2016, 11, 6), datetime(2016, 11, 8) - self.assertEqual(result, expected) - - result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59") - expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59) - self.assertEqual(result, expected) - - result = get_since_until("Last year") - expected = datetime(2015, 11, 7), datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until("Last quarter") - expected = datetime(2016, 8, 7), datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until("Last 5 months") - expected = datetime(2016, 6, 7), datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until("Next 5 months") - expected = datetime(2016, 11, 7), datetime(2017, 4, 7) - self.assertEqual(result, expected) - - result = get_since_until(since="5 days") - expected = datetime(2016, 11, 2), datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until(since="5 days ago", until="tomorrow") - expected = datetime(2016, 11, 2), datetime(2016, 11, 8) - self.assertEqual(result, expected) - - result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day") - expected = datetime(2016, 11, 5), datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until(time_range="5 days : now") - expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10) - self.assertEqual(result, expected) - - result = get_since_until("Last week", relative_end="now") - expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10) - self.assertEqual(result, expected) - - result = get_since_until("Last week", relative_start="now") - expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7) - self.assertEqual(result, expected) - - result = get_since_until("Last week", relative_start="now", relative_end="now") - expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10) - self.assertEqual(result, expected) - - result = get_since_until("previous calendar week") - expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0) - self.assertEqual(result, expected) - - result = get_since_until("previous calendar month") - expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0) - self.assertEqual(result, expected) - - result = get_since_until("previous calendar year") - expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0) - self.assertEqual(result, expected) - - with self.assertRaises(ValueError): - get_since_until(time_range="tomorrow : yesterday") - - @patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime) - def test_datetime_eval(self): - result = datetime_eval("datetime('now')") - expected = datetime(2016, 11, 7, 9, 30, 10) - self.assertEqual(result, expected) - - result = datetime_eval("datetime('today' )") - expected = datetime(2016, 11, 7) - self.assertEqual(result, expected) - - # Parse compact arguments spelling - result = datetime_eval("dateadd(datetime('today'),1,year,)") - expected = datetime(2017, 11, 7) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('today'), -2, year)") - expected = datetime(2014, 11, 7) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('today'), 2, quarter)") - expected = datetime(2017, 5, 7) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('today'), 3, month)") - expected = datetime(2017, 2, 7) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('today'), -3, week)") - expected = datetime(2016, 10, 17) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('today'), 3, day)") - expected = datetime(2016, 11, 10) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('now'), 3, hour)") - expected = datetime(2016, 11, 7, 12, 30, 10) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('now'), 40, minute)") - expected = datetime(2016, 11, 7, 10, 10, 10) - self.assertEqual(result, expected) - - result = datetime_eval("dateadd(datetime('now'), -11, second)") - expected = datetime(2016, 11, 7, 9, 29, 59) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), year)") - expected = datetime(2016, 1, 1, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), month)") - expected = datetime(2016, 11, 1, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), day)") - expected = datetime(2016, 11, 7, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), week)") - expected = datetime(2016, 11, 7, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), hour)") - expected = datetime(2016, 11, 7, 9, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), minute)") - expected = datetime(2016, 11, 7, 9, 30, 0) - self.assertEqual(result, expected) - - result = datetime_eval("datetrunc(datetime('now'), second)") - expected = datetime(2016, 11, 7, 9, 30, 10) - self.assertEqual(result, expected) - - result = datetime_eval("lastday(datetime('now'), year)") - expected = datetime(2016, 12, 31, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("lastday(datetime('today'), month)") - expected = datetime(2016, 11, 30, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("holiday('Christmas')") - expected = datetime(2016, 12, 25, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))") - expected = datetime(2018, 9, 3, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval( - "holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')" - ) - expected = datetime(2018, 12, 26, 0, 0, 0) - self.assertEqual(result, expected) - - result = datetime_eval( - "lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)" - ) - expected = datetime(2018, 2, 28, 0, 0, 0) - self.assertEqual(result, expected) - @patch("superset.utils.core.to_adhoc", mock_to_adhoc) def test_convert_legacy_filters_into_adhoc_where(self): form_data = {"where": "a = 1"}