diff --git a/requirements/base.txt b/requirements/base.txt index 2edf141431..2d5f8247d9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,107 +5,301 @@ # # pip-compile-multi # --e file:. # via -r requirements/base.in -aiohttp==3.7.2 # via slackclient -alembic==1.4.3 # via flask-migrate -amqp==2.6.1 # via kombu -apispec[yaml]==3.3.2 # via flask-appbuilder -async-timeout==3.0.1 # via aiohttp -attrs==20.2.0 # via aiohttp, jsonschema -babel==2.8.0 # via flask-babel -backoff==1.10.0 # via apache-superset -billiard==3.6.3.0 # via celery -bleach==3.2.1 # via apache-superset -brotli==1.0.9 # via flask-compress -cachelib==0.1.1 # via apache-superset -celery==4.4.7 # via apache-superset -cffi==1.14.3 # via cryptography -chardet==3.0.4 # via aiohttp -click==7.1.2 # via apache-superset, flask, flask-appbuilder -colorama==0.4.4 # via apache-superset, flask-appbuilder -contextlib2==0.6.0.post1 # via apache-superset -convertdate==2.3.0 # via holidays -cron-descriptor==1.2.24 # via apache-superset -croniter==0.3.36 # via apache-superset -cryptography==3.2.1 # via apache-superset -decorator==4.4.2 # via retry -defusedxml==0.6.0 # via python3-openid -dnspython==2.0.0 # via email-validator -email-validator==1.1.1 # via flask-appbuilder -flask-appbuilder==3.1.1 # via apache-superset -flask-babel==1.0.0 # via flask-appbuilder -flask-caching==1.9.0 # via apache-superset -flask-compress==1.8.0 # via apache-superset -flask-jwt-extended==3.24.1 # via flask-appbuilder -flask-login==0.4.1 # via flask-appbuilder -flask-migrate==2.5.3 # via apache-superset -flask-openid==1.2.5 # via flask-appbuilder -flask-sqlalchemy==2.4.4 # via flask-appbuilder, flask-migrate -flask-talisman==0.7.0 # via apache-superset -flask-wtf==0.14.3 # via apache-superset, flask-appbuilder -flask==1.1.2 # via apache-superset, flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf -geographiclib==1.50 # via geopy -geopy==2.0.0 # via apache-superset -gunicorn==20.0.4 # via apache-superset -holidays==0.10.3 # via apache-superset -humanize==3.1.0 # via apache-superset -idna==2.10 # via email-validator, yarl -importlib-metadata==2.1.1 # via -r requirements/base.in, jsonschema, kombu, markdown -isodate==0.6.0 # via apache-superset -itsdangerous==1.1.0 # via flask, flask-wtf -jinja2==2.11.3 # via flask, flask-babel -jsonschema==3.2.0 # via flask-appbuilder -kombu==4.6.11 # via celery -korean-lunar-calendar==0.2.1 # via holidays -mako==1.1.3 # via alembic -markdown==3.3.3 # via apache-superset -markupsafe==1.1.1 # via jinja2, mako, wtforms -marshmallow-enum==1.5.1 # via flask-appbuilder -marshmallow-sqlalchemy==0.23.1 # via flask-appbuilder -marshmallow==3.9.0 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy -msgpack==1.0.0 # via apache-superset -multidict==5.0.0 # via aiohttp, yarl -natsort==7.0.1 # via croniter -numpy==1.19.4 # via pandas, pyarrow -packaging==20.4 # via bleach -pandas==1.1.4 # via apache-superset -parsedatetime==2.6 # via apache-superset -pathlib2==2.3.5 # via apache-superset -pgsanity==0.2.9 # via apache-superset -polyline==1.4.0 # via apache-superset -prison==0.1.3 # via flask-appbuilder -py==1.9.0 # via retry -pyarrow==1.0.1 # via apache-superset -pycparser==2.20 # via cffi -pyjwt==1.7.1 # via apache-superset, flask-appbuilder, flask-jwt-extended -pymeeus==0.3.7 # via convertdate -pyparsing==2.4.7 # via apache-superset, packaging -pyrsistent==0.16.1 # via -r requirements/base.in, jsonschema -python-dateutil==2.8.1 # via alembic, apache-superset, croniter, flask-appbuilder, holidays, pandas -python-dotenv==0.15.0 # via apache-superset -python-editor==1.0.4 # via alembic -python-geohash==0.8.5 # via apache-superset -python3-openid==3.2.0 # via flask-openid -pytz==2020.4 # via babel, celery, convertdate, flask-babel, pandas -pyyaml==5.3.1 # via apache-superset, apispec -redis==3.5.3 # via apache-superset -retry==0.9.2 # via apache-superset -selenium==3.141.0 # via apache-superset -simplejson==3.17.2 # via apache-superset -six==1.15.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json -slackclient==2.5.0 # via apache-superset -sqlalchemy-utils==0.36.8 # via apache-superset, flask-appbuilder -sqlalchemy==1.3.20 # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -sqlparse==0.3.0 # via apache-superset -typing-extensions==3.7.4.3 # via aiohttp, apache-superset, yarl -urllib3==1.25.11 # via selenium -vine==1.3.0 # via amqp, celery -webencodings==0.5.1 # via bleach -werkzeug==1.0.1 # via flask, flask-jwt-extended -wtforms-json==0.3.3 # via apache-superset -wtforms==2.3.3 # via flask-wtf, wtforms-json -yarl==1.6.2 # via aiohttp -zipp==3.4.0 # via importlib-metadata +-e file:. +# via -r requirements/base.in +aiohttp==3.7.2 +# via slackclient +alembic==1.4.3 +# via flask-migrate +amqp==2.6.1 +# via kombu +apispec[yaml]==3.3.2 +# via flask-appbuilder +async-timeout==3.0.1 +# via aiohttp +attrs==20.2.0 +# via +# aiohttp +# jsonschema +babel==2.8.0 +# via flask-babel +backoff==1.10.0 +# via apache-superset +billiard==3.6.3.0 +# via celery +bleach==3.2.1 +# via apache-superset +brotli==1.0.9 +# via flask-compress +cachelib==0.1.1 +# via apache-superset +celery==4.4.7 +# via apache-superset +cffi==1.14.3 +# via cryptography +chardet==3.0.4 +# via aiohttp +click==7.1.2 +# via +# apache-superset +# flask +# flask-appbuilder +colorama==0.4.4 +# via +# apache-superset +# flask-appbuilder +contextlib2==0.6.0.post1 +# via apache-superset +convertdate==2.3.0 +# via holidays +cron-descriptor==1.2.24 +# via apache-superset +croniter==0.3.36 +# via apache-superset +cryptography==3.2.1 +# via apache-superset +decorator==4.4.2 +# via retry +defusedxml==0.6.0 +# via python3-openid +dnspython==2.0.0 +# via email-validator +email-validator==1.1.1 +# via flask-appbuilder +flask-appbuilder==3.1.1 +# via apache-superset +flask-babel==1.0.0 +# via flask-appbuilder +flask-caching==1.9.0 +# via apache-superset +flask-compress==1.8.0 +# via apache-superset +flask-jwt-extended==3.24.1 +# via flask-appbuilder +flask-login==0.4.1 +# via flask-appbuilder +flask-migrate==2.5.3 +# via apache-superset +flask-openid==1.2.5 +# via flask-appbuilder +flask-sqlalchemy==2.4.4 +# via +# flask-appbuilder +# flask-migrate +flask-talisman==0.7.0 +# via apache-superset +flask-wtf==0.14.3 +# via +# apache-superset +# flask-appbuilder +flask==1.1.2 +# via +# apache-superset +# flask-appbuilder +# flask-babel +# flask-caching +# flask-compress +# flask-jwt-extended +# flask-login +# flask-migrate +# flask-openid +# flask-sqlalchemy +# flask-wtf +geographiclib==1.50 +# via geopy +geopy==2.0.0 +# via apache-superset +gunicorn==20.0.4 +# via apache-superset +holidays==0.10.3 +# via apache-superset +humanize==3.1.0 +# via apache-superset +idna==2.10 +# via +# email-validator +# yarl +importlib-metadata==2.1.1 +# via +# -r requirements/base.in +# jsonschema +# kombu +# markdown +isodate==0.6.0 +# via apache-superset +itsdangerous==1.1.0 +# via +# flask +# flask-wtf +jinja2==2.11.3 +# via +# flask +# flask-babel +jsonschema==3.2.0 +# via flask-appbuilder +kombu==4.6.11 +# via celery +korean-lunar-calendar==0.2.1 +# via holidays +mako==1.1.3 +# via alembic +markdown==3.3.3 +# via apache-superset +markupsafe==1.1.1 +# via +# jinja2 +# mako +# wtforms +marshmallow-enum==1.5.1 +# via flask-appbuilder +marshmallow-sqlalchemy==0.23.1 +# via flask-appbuilder +marshmallow==3.9.0 +# via +# flask-appbuilder +# marshmallow-enum +# marshmallow-sqlalchemy +msgpack==1.0.0 +# via apache-superset +multidict==5.0.0 +# via +# aiohttp +# yarl +natsort==7.0.1 +# via croniter +numpy==1.19.4 +# via +# pandas +# pyarrow +packaging==20.4 +# via bleach +pandas==1.2.2 +# via apache-superset +parsedatetime==2.6 +# via apache-superset +pathlib2==2.3.5 +# via apache-superset +pgsanity==0.2.9 +# via apache-superset +polyline==1.4.0 +# via apache-superset +prison==0.1.3 +# via flask-appbuilder +py==1.9.0 +# via retry +pyarrow==3.0.0 +# via apache-superset +pycparser==2.20 +# via cffi +pyjwt==1.7.1 +# via +# apache-superset +# flask-appbuilder +# flask-jwt-extended +pymeeus==0.3.7 +# via convertdate +pyparsing==2.4.7 +# via +# apache-superset +# packaging +pyrsistent==0.16.1 +# via +# -r requirements/base.in +# jsonschema +python-dateutil==2.8.1 +# via +# alembic +# apache-superset +# croniter +# flask-appbuilder +# holidays +# pandas +python-dotenv==0.15.0 +# via apache-superset +python-editor==1.0.4 +# via alembic +python-geohash==0.8.5 +# via apache-superset +python3-openid==3.2.0 +# via flask-openid +pytz==2020.4 +# via +# babel +# celery +# convertdate +# flask-babel +# pandas +pyyaml==5.3.1 +# via +# apache-superset +# apispec +redis==3.5.3 +# via apache-superset +retry==0.9.2 +# via apache-superset +selenium==3.141.0 +# via apache-superset +simplejson==3.17.2 +# via apache-superset +six==1.15.0 +# via +# bleach +# cryptography +# flask-jwt-extended +# flask-talisman +# holidays +# isodate +# jsonschema +# packaging +# pathlib2 +# polyline +# prison +# pyrsistent +# python-dateutil +# sqlalchemy-utils +# wtforms-json +slackclient==2.5.0 +# via apache-superset +sqlalchemy-utils==0.36.8 +# via +# apache-superset +# flask-appbuilder +sqlalchemy==1.3.20 +# via +# alembic +# apache-superset +# flask-sqlalchemy +# marshmallow-sqlalchemy +# sqlalchemy-utils +sqlparse==0.3.0 +# via apache-superset +typing-extensions==3.7.4.3 +# via +# aiohttp +# apache-superset +# yarl +urllib3==1.25.11 +# via selenium +vine==1.3.0 +# via +# amqp +# celery +webencodings==0.5.1 +# via bleach +werkzeug==1.0.1 +# via +# flask +# flask-jwt-extended +wtforms-json==0.3.3 +# via apache-superset +wtforms==2.3.3 +# via +# flask-wtf +# wtforms-json +yarl==1.6.2 +# via aiohttp +zipp==3.4.0 +# via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index 78be764284..8cba022303 100644 --- a/setup.py +++ b/setup.py @@ -87,7 +87,7 @@ setup( "isodate", "markdown>=3.0", "msgpack>=1.0.0, <1.1", - "pandas>=1.1.2, <1.2", + "pandas>=1.2.2, <1.3", "parsedatetime", "pathlib2", "pgsanity", @@ -95,7 +95,7 @@ setup( "python-dateutil", "python-dotenv", "python-geohash", - "pyarrow>=1.0.1, <1.1", + "pyarrow>=3.0.0, <3.1", "pyyaml>=5.1", "PyJWT>=1.7.1, <2", "redis", diff --git a/superset/common/query_context.py b/superset/common/query_context.py index 58b5238cbc..a740af5490 100644 --- a/superset/common/query_context.py +++ b/superset/common/query_context.py @@ -119,7 +119,7 @@ class QueryContext: # If the datetime format is unix, the parse will use the corresponding # parsing logic if not df.empty: - df = normalize_dttm_col( + normalize_dttm_col( df=df, timestamp_format=timestamp_format, offset=self.datasource.offset, diff --git a/superset/utils/core.py b/superset/utils/core.py index 4efad98bcc..4c48f73d08 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -1604,10 +1604,9 @@ def normalize_dttm_col( timestamp_format: Optional[str], offset: int, time_shift: Optional[timedelta], -) -> pd.DataFrame: +) -> None: if DTTM_ALIAS not in df.columns: - return df - df = df.copy() + return if timestamp_format in ("epoch_s", "epoch_ms"): dttm_col = df[DTTM_ALIAS] if is_numeric_dtype(dttm_col): @@ -1627,4 +1626,3 @@ def normalize_dttm_col( df[DTTM_ALIAS] += timedelta(hours=offset) if time_shift is not None: df[DTTM_ALIAS] += time_shift - return df diff --git a/superset/viz.py b/superset/viz.py index b403899e54..1183418877 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -284,7 +284,7 @@ class BaseViz: # If the datetime format is unix, the parse will use the corresponding # parsing logic. if not df.empty: - df = utils.normalize_dttm_col( + utils.normalize_dttm_col( df=df, timestamp_format=timestamp_format, offset=self.datasource.offset, diff --git a/tests/utils_tests.py b/tests/utils_tests.py index 9fed17251b..d810c40f2d 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -23,7 +23,7 @@ import hashlib import json import os import re -from typing import Any, Tuple, List +from typing import Any, Tuple, List, Optional from unittest.mock import Mock, patch from tests.fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices @@ -1135,28 +1135,38 @@ class TestUtils(SupersetTestCase): assert extract_dataframe_dtypes(df) == [col[1] for col in cols] def test_normalize_dttm_col(self): + def normalize_col( + df: pd.DataFrame, + timestamp_format: Optional[str], + offset: int, + time_shift: Optional[timedelta], + ) -> pd.DataFrame: + df = df.copy() + normalize_dttm_col(df, timestamp_format, offset, time_shift) + return df + ts = pd.Timestamp(2021, 2, 15, 19, 0, 0, 0) df = pd.DataFrame([{"__timestamp": ts, "a": 1}]) # test regular (non-numeric) format - assert normalize_dttm_col(df, None, 0, None)[DTTM_ALIAS][0] == ts - assert normalize_dttm_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts - assert normalize_dttm_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts + assert normalize_col(df, None, 0, None)[DTTM_ALIAS][0] == ts + assert normalize_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts + assert normalize_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts # test offset - assert normalize_dttm_col(df, None, 1, None)[DTTM_ALIAS][0] == pd.Timestamp( + assert normalize_col(df, None, 1, None)[DTTM_ALIAS][0] == pd.Timestamp( 2021, 2, 15, 20, 0, 0, 0 ) # test offset and timedelta - assert normalize_dttm_col(df, None, 1, timedelta(minutes=30))[DTTM_ALIAS][ + assert normalize_col(df, None, 1, timedelta(minutes=30))[DTTM_ALIAS][ 0 ] == pd.Timestamp(2021, 2, 15, 20, 30, 0, 0) # test numeric epoch_s format df = pd.DataFrame([{"__timestamp": ts.timestamp(), "a": 1}]) - assert normalize_dttm_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts + assert normalize_col(df, "epoch_s", 0, None)[DTTM_ALIAS][0] == ts # test numeric epoch_ms format df = pd.DataFrame([{"__timestamp": ts.timestamp() * 1000, "a": 1}]) - assert normalize_dttm_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts + assert normalize_col(df, "epoch_ms", 0, None)[DTTM_ALIAS][0] == ts