chore: bump python dependencies (#9786)

* chore: bump pyhton dependencies

* bump flask-testing

* Add note to UPDATING and bump flask-testing in requirements-dev.txt

* fix old sqlparse test case that is fixed with sqlparse==0.3.1

* Fix migration and sqlparse set test

* downgrade alembic

* Downgrade sqlalchemy-utils

* pin sqlalchemy-utils due to regression in 0.36.5

* Refine comment in setup.py

* make cachelib an explicit requirement

CONTRIBUTING.md

@@ -927,7 +927,7 @@ To do this, you'll need to:
   but perfect for testing (stores cache in `/tmp`)
 
   ```python
-  from werkzeug.contrib.cache import FileSystemCache
+  from cachelib.file import FileSystemCache
   RESULTS_BACKEND = FileSystemCache('/tmp/sqllab')
   ```
 

UPDATING.md

@@ -23,6 +23,8 @@ assists people when migrating to a new version.
 
 ## Next
 
+* [9786](https://github.com/apache/incubator-superset/pull/9786): with the upgrade of `werkzeug` from version `0.16.0` to `1.0.1`, the `werkzeug.contrib.cache` module has been moved to a standalone package [cachelib](https://pypi.org/project/cachelib/). For example, to import the `RedisCache` class, please use the following import: `from cachelib.redis import RedisCache`.
+
 * [9572](https://github.com/apache/incubator-superset/pull/9572): a change which by defau;t means that the Jinja `current_user_id`, `current_username`, and `url_param` context calls no longer need to be wrapped via `cache_key_wrapper` in order to be included in the cache key. The `cache_key_wrapper` function should only be required for Jinja add-ons.
 
 * [8867](https://github.com/apache/incubator-superset/pull/8867): a change which adds the `tmp_schema_name` column to the `query` table which requires locking the table. Given the `query` table is heavily used performance may be degraded during the migration. Scheduled downtime may be advised.

docker/pythonpath_dev/superset_config.py

@@ -25,7 +25,7 @@
 import logging
 import os
 
-from werkzeug.contrib.cache import FileSystemCache
+from cachelib.file import FileSystemCache
 
 logger = logging.getLogger()
 

docs/installation.rst

@@ -1056,7 +1056,7 @@ have the same configuration.
     celery beat --app=superset.tasks.celery_app:app
 
 To setup a result backend, you need to pass an instance of a derivative
-of ``werkzeug.contrib.cache.BaseCache`` to the ``RESULTS_BACKEND``
+of ``from cachelib.base.BaseCache`` to the ``RESULTS_BACKEND``
 configuration key in your ``superset_config.py``. It's possible to use
 Memcached, Redis, S3 (https://pypi.python.org/pypi/s3werkzeugcache),
 memory or the file system (in a single server-type setup or for testing),
@@ -1072,7 +1072,7 @@ look something like:
     RESULTS_BACKEND = S3Cache(S3_CACHE_BUCKET, S3_CACHE_KEY_PREFIX)
 
     # On Redis
-    from werkzeug.contrib.cache import RedisCache
+    from cachelib.redis import RedisCache
     RESULTS_BACKEND = RedisCache(
         host='localhost', port=6379, key_prefix='superset_results')
 

requirements-dev.txt

@@ -17,20 +17,20 @@
 black==19.10b0
 coverage==4.5.3
 flask-cors==3.0.7
-flask-testing==0.7.1
+flask-testing==0.8.0
 ipdb==0.12
 isort==4.3.21
 mypy==0.770
 nose==1.3.7
-pip-tools==4.5.1
+pip-tools==5.1.2
 pre-commit==1.17.0
 psycopg2-binary==2.7.5
 pycodestyle==2.5.0
-pydruid==0.5.7
+pydruid==0.5.9
-pyhive==0.6.1
+pyhive==0.6.2
 pylint==1.9.2
-redis==3.2.1
+redis==3.5.1
-requests==2.22.0
+requests==2.23.0
 statsd==3.3.0
 tox==3.11.1
 pillow==7.0.0

requirements.txt

@@ -4,58 +4,60 @@
 #
 #    pip-compile --output-file=requirements.txt setup.py
 #
-alembic==1.3.2            # via flask-migrate
+alembic==1.4.2            # via flask-migrate
 amqp==2.5.2               # via kombu
 apispec[yaml]==1.3.3      # via flask-appbuilder
 attrs==19.3.0             # via jsonschema
 babel==2.8.0              # via flask-babel
 backoff==1.10.0           # via apache-superset (setup.py)
-billiard==3.6.1.0         # via celery
+billiard==3.6.3.0         # via celery
-bleach==3.1.0             # via apache-superset (setup.py)
+bleach==3.1.5             # via apache-superset (setup.py)
-celery==4.4.0             # via apache-superset (setup.py)
+brotli==1.0.7             # via flask-compress
-cffi==1.13.2              # via cryptography
+cachelib==0.1             # via apache-superset (setup.py)
-click==7.1.1              # via apache-superset (setup.py), flask, flask-appbuilder
+celery==4.4.2             # via apache-superset (setup.py)
+cffi==1.14.0              # via cryptography
+click==7.1.2              # via apache-superset (setup.py), flask, flask-appbuilder
 colorama==0.4.3           # via apache-superset (setup.py), flask-appbuilder
 contextlib2==0.6.0.post1  # via apache-superset (setup.py)
 croniter==0.3.31          # via apache-superset (setup.py)
-cryptography==2.8         # via apache-superset (setup.py)
+cryptography==2.9.2       # via apache-superset (setup.py)
 dataclasses==0.6          # via apache-superset (setup.py)
-decorator==4.4.1          # via retry
+decorator==4.4.2          # via retry
 defusedxml==0.6.0         # via python3-openid
 dnspython==1.16.0         # via email-validator
-email-validator==1.0.5    # via flask-appbuilder
+email-validator==1.1.0    # via flask-appbuilder
 flask-appbuilder==2.3.4   # via apache-superset (setup.py)
 flask-babel==1.0.0        # via flask-appbuilder
 flask-caching==1.8.0      # via apache-superset (setup.py)
-flask-compress==1.4.0     # via apache-superset (setup.py)
+flask-compress==1.5.0     # via apache-superset (setup.py)
 flask-jwt-extended==3.24.1  # via flask-appbuilder
 flask-login==0.4.1        # via flask-appbuilder
-flask-migrate==2.5.2      # via apache-superset (setup.py)
+flask-migrate==2.5.3      # via apache-superset (setup.py)
 flask-openid==1.2.5       # via flask-appbuilder
 flask-sqlalchemy==2.4.1   # via flask-appbuilder, flask-migrate
 flask-talisman==0.7.0     # via apache-superset (setup.py)
-flask-wtf==0.14.2         # via apache-superset (setup.py), flask-appbuilder
+flask-wtf==0.14.3         # via apache-superset (setup.py), flask-appbuilder
-flask==1.1.1              # via apache-superset (setup.py), flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf
+flask==1.1.2              # via apache-superset (setup.py), flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf
 geographiclib==1.50       # via geopy
-geopy==1.21.0             # via apache-superset (setup.py)
+geopy==1.22.0             # via apache-superset (setup.py)
 gunicorn==20.0.4          # via apache-superset (setup.py)
-humanize==0.5.1           # via apache-superset (setup.py)
+humanize==2.4.0           # via apache-superset (setup.py)
 idna==2.9                 # via email-validator
-importlib-metadata==1.4.0  # via jsonschema, kombu
+importlib-metadata==1.6.0  # via jsonschema, kombu, markdown
 isodate==0.6.0            # via apache-superset (setup.py)
-itsdangerous==1.1.0       # via flask
+itsdangerous==1.1.0       # via flask, flask-wtf
-jinja2==2.10.3            # via flask, flask-babel
+jinja2==2.11.2            # via flask, flask-babel
 jsonschema==3.2.0         # via flask-appbuilder
-kombu==4.6.7              # via celery
+kombu==4.6.8              # via celery
-mako==1.1.1               # via alembic
+mako==1.1.2               # via alembic
-markdown==3.1.1           # via apache-superset (setup.py)
+markdown==3.2.2           # via apache-superset (setup.py)
-markupsafe==1.1.1         # via jinja2, mako
+markupsafe==1.1.1         # via jinja2, mako, wtforms
 marshmallow-enum==1.5.1   # via flask-appbuilder
-marshmallow-sqlalchemy==0.21.0  # via flask-appbuilder
+marshmallow-sqlalchemy==0.23.0  # via flask-appbuilder
-marshmallow==2.19.5       # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
+marshmallow==2.21.0       # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
-more-itertools==8.1.0     # via zipp
+msgpack==1.0.0            # via apache-superset (setup.py)
-msgpack==0.6.2            # via apache-superset (setup.py)
+numpy==1.18.4             # via pandas, pyarrow
-numpy==1.18.1             # via pandas, pyarrow
+packaging==20.3           # via bleach
 pandas==1.0.3             # via apache-superset (setup.py)
 parsedatetime==2.5        # via apache-superset (setup.py)
 pathlib2==2.3.5           # via apache-superset (setup.py)
@@ -63,30 +65,31 @@ polyline==1.4.0           # via apache-superset (setup.py)
 prison==0.1.3             # via flask-appbuilder
 py==1.8.1                 # via retry
 pyarrow==0.17.0           # via apache-superset (setup.py)
-pycparser==2.19           # via cffi
+pycparser==2.20           # via cffi
 pyjwt==1.7.1              # via flask-appbuilder, flask-jwt-extended
-pyrsistent==0.15.7        # via jsonschema
+pyparsing==2.4.7          # via packaging
+pyrsistent==0.16.0        # via jsonschema
 python-dateutil==2.8.1    # via alembic, apache-superset (setup.py), croniter, flask-appbuilder, pandas
-python-dotenv==0.10.5     # via apache-superset (setup.py)
+python-dotenv==0.13.0     # via apache-superset (setup.py)
 python-editor==1.0.4      # via alembic
 python-geohash==0.8.5     # via apache-superset (setup.py)
 python3-openid==3.1.0     # via flask-openid
-pytz==2019.3              # via babel, celery, flask-babel, pandas
+pytz==2020.1              # via babel, celery, flask-babel, pandas
-pyyaml==5.3               # via apache-superset (setup.py), apispec
+pyyaml==5.3.1             # via apache-superset (setup.py), apispec
 retry==0.9.2              # via apache-superset (setup.py)
 selenium==3.141.0         # via apache-superset (setup.py)
 simplejson==3.17.0        # via apache-superset (setup.py)
-six==1.14.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
+six==1.14.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
-sqlalchemy-utils==0.36.1  # via apache-superset (setup.py), flask-appbuilder
+sqlalchemy-utils==0.36.4  # via apache-superset (setup.py), flask-appbuilder
 sqlalchemy==1.3.16        # via alembic, apache-superset (setup.py), flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
-sqlparse==0.3.0           # via apache-superset (setup.py)
+sqlparse==0.3.1           # via apache-superset (setup.py)
-urllib3==1.25.8           # via selenium
+urllib3==1.25.9           # via selenium
 vine==1.3.0               # via amqp, celery
 webencodings==0.5.1       # via bleach
-werkzeug==0.16.0          # via flask, flask-jwt-extended
+werkzeug==1.0.1           # via flask, flask-jwt-extended
 wtforms-json==0.3.3       # via apache-superset (setup.py)
-wtforms==2.2.1            # via flask-wtf, wtforms-json
+wtforms==2.3.1            # via flask-wtf, wtforms-json
-zipp==2.0.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

setup.cfg

@@ -45,7 +45,7 @@ combine_as_imports = true
 include_trailing_comma = true
 line_length = 88
 known_first_party = superset
-known_third_party =alembic,apispec,backoff,bleach,celery,click,colorama,contextlib2,croniter,cryptography,dataclasses,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,geohash,geopy,humanize,isodate,jinja2,markdown,markupsafe,marshmallow,msgpack,numpy,pandas,parsedatetime,pathlib2,polyline,prison,pyarrow,pyhive,pytz,retry,selenium,setuptools,simplejson,sphinx_rtd_theme,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml
+known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,croniter,cryptography,dataclasses,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,geohash,geopy,humanize,isodate,jinja2,markdown,markupsafe,marshmallow,msgpack,numpy,pandas,parsedatetime,pathlib2,polyline,prison,pyarrow,pyhive,pytz,retry,selenium,setuptools,simplejson,sphinx_rtd_theme,sqlalchemy,sqlalchemy_utils,sqlparse,werkzeug,wtforms,wtforms_json,yaml
 multi_line_output = 3
 order_by_type = false
 

setup.py

@@ -69,6 +69,7 @@ setup(
     install_requires=[
         "backoff>=1.8.0",
         "bleach>=3.0.2, <4.0.0",
+        "cachelib>=0.1,<0.2",
         "celery>=4.3.0, <5.0.0, !=4.4.1",
         "click<8",
         "colorama",
@@ -88,7 +89,7 @@ setup(
         "humanize",
         "isodate",
         "markdown>=3.0",
-        "msgpack>=0.6.1, <0.7.0",
+        "msgpack>=1.0.0, <1.1",
         "pandas>=1.0.3, <1.1",
         "parsedatetime",
         "pathlib2",
@@ -102,7 +103,10 @@ setup(
         "selenium>=3.141.0",
         "simplejson>=3.15.0",
         "sqlalchemy>=1.3.16, <2.0",
-        "sqlalchemy-utils>=0.33.2",
+        # Breaking change in sqlalchemy-utils==0.36.6, upgrading will probably
+        # require a migration on EncryptedType columns. For more information, see
+        # https://github.com/kvesteri/sqlalchemy-utils/issues/444
+        "sqlalchemy-utils>=0.33.2,<0.36.5",
         "sqlparse>=0.3.0, <0.4",
         "wtforms-json",
     ],
@@ -130,5 +134,5 @@ setup(
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
     ],
-    tests_require=["flask-testing==0.7.1"],
+    tests_require=["flask-testing==0.8.0"],
 )

superset/config.py

@@ -568,7 +568,7 @@ SQLLAB_CTAS_SCHEMA_NAME_FUNC: Optional[
     Callable[["Database", "models.User", str, str], str]
 ] = None
 
-# An instantiated derivative of werkzeug.contrib.cache.BaseCache
+# An instantiated derivative of cachelib.base.BaseCache
 # if enabled, it can be used to store the results of long-running queries
 # in SQL Lab by using the "Run Async" button/feature
 RESULTS_BACKEND = None

tests/sql_parse_tests.py

@@ -187,8 +187,7 @@ class SupersetTestCase(unittest.TestCase):
     # SHOW TABLES ((FROM | IN) qualifiedName)? (LIKE pattern=STRING)?
     def test_show_tables(self):
         query = "SHOW TABLES FROM s1 like '%order%'"
-        # TODO: figure out what should code do here
+        self.assertEqual(set(), self.extract_tables(query))
-        self.assertEqual({Table("s1")}, self.extract_tables(query))
 
     # SHOW COLUMNS (FROM | IN) qualifiedName
     def test_show_columns(self):

tests/superset_test_config_thumbnails.py

@@ -17,8 +17,8 @@
 # type: ignore
 from copy import copy
 
+from cachelib.redis import RedisCache
 from flask import Flask
-from werkzeug.contrib.cache import RedisCache
 
 from superset.config import *  # type: ignore
 

tests/thumbnails_tests.py

@@ -42,7 +42,7 @@ class CeleryStartMixin:
     @classmethod
     def setUpClass(cls):
         with app.app_context():
-            from werkzeug.contrib.cache import RedisCache
+            from cachelib.redis import RedisCache
 
             class CeleryConfig(object):
                 BROKER_URL = "redis://localhost"