feat: upgrade docker image to py38 and add support for py39 (#16889)

* feat: upgrade docker image to py38 and add support for py39

* update required tests

.github/workflows/superset-e2e.yml

@@ -67,7 +67,7 @@ jobs:
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.8"
       - name: OS dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

.github/workflows/superset-helm-lint.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.8
 
       - name: Set up chart-testing
         uses: ./.github/actions/chart-testing-action

.github/workflows/superset-python-integrationtest.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -77,7 +77,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8, 3.9]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config
@@ -141,7 +141,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     env:
       PYTHONPATH: ${{ github.workspace }}
       SUPERSET_CONFIG: tests.integration_tests.superset_test_config

.github/workflows/superset-python-misc.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -50,7 +50,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2
@@ -76,7 +76,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2

.github/workflows/superset-python-unittest.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7,3.8]
+        python-version: [3.8, 3.9]
     env:
       PYTHONPATH: ${{ github.workspace }}
     steps:

.github/workflows/superset-translations.yml

@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
         uses: actions/checkout@v2

Dockerfile

@@ -18,7 +18,7 @@
 ######################################################################
 # PY stage that simply does a pip install on our requirements
 ######################################################################
-ARG PY_VER=3.7.9
+ARG PY_VER=3.8.12
 FROM python:${PY_VER} AS superset-py
 
 RUN mkdir /app \
@@ -73,7 +73,7 @@ RUN cd /app/superset-frontend \
 ######################################################################
 # Final lean image...
 ######################################################################
-ARG PY_VER=3.7.9
+ARG PY_VER=3.8.12
 FROM python:${PY_VER} AS lean
 
 ENV LANG=C.UTF-8 \
@@ -94,7 +94,7 @@ RUN mkdir -p ${PYTHONPATH} \
             libpq-dev \
         && rm -rf /var/lib/apt/lists/*
 
-COPY --from=superset-py /usr/local/lib/python3.7/site-packages/ /usr/local/lib/python3.7/site-packages/
+COPY --from=superset-py /usr/local/lib/python3.8/site-packages/ /usr/local/lib/python3.8/site-packages/
 # Copying site-packages doesn't move the CLIs, so let's copy them one by one
 COPY --from=superset-py /usr/local/bin/gunicorn /usr/local/bin/celery /usr/local/bin/flask /usr/bin/
 COPY --from=superset-node /app/superset/static/assets /app/superset/static/assets

Makefile

@@ -15,8 +15,8 @@
 # limitations under the License.
 #
 
-# Python version installed; we need 3.8 or 3.7
-PYTHON=`command -v python3.8 || command -v python3.7`
+# Python version installed; we need 3.7-3.9
+PYTHON=`command -v python3.9 || command -v python3.8 || command -v python3.7`
 
 .PHONY: install superset venv pre-commit
 
@@ -62,7 +62,7 @@ update-js:
 
 venv:
 	# Create a virtual environment and activate it (recommended)
-	if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.7 or 3.8 installed"; exit 1; fi
+	if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.7, 3.8 or 3.9 installed"; exit 1; fi
 	test -d venv || ${PYTHON} -m venv venv # setup a python3 virtualenv
 	. venv/bin/activate
 

RELEASING/Dockerfile.from_local_tarball

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
 

RELEASING/Dockerfile.from_svn_tarball

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset
 

RELEASING/Dockerfile.make_docs

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 ARG VERSION
 
 RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/superset.git /superset

RELEASING/Dockerfile.make_tarball

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM python:3.7-buster
+FROM python:3.8-buster
 
 RUN apt-get update -y
 RUN apt-get install -y jq

docs/src/pages/docs/installation/installing_scratch.mdx

@@ -65,7 +65,7 @@ We don't recommend using the system installed Python. Instead, first install the
 brew install readline pkg-config libffi openssl mysql postgres
 ```
 
-You should install a recent version of Python (Superset uses 3.7.9). We'd recommend using a Python version manager like [pyenv](https://github.com/pyenv/pyenv) (and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
+You should install a recent version of Python (the official docker image uses 3.8.12). We'd recommend using a Python version manager like [pyenv](https://github.com/pyenv/pyenv) (and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
 
 Let's also make sure we have the latest version of `pip` and `setuptools`:
 

requirements/base.txt

@@ -257,7 +257,7 @@ sqlalchemy==1.3.24
     #   flask-sqlalchemy
     #   marshmallow-sqlalchemy
     #   sqlalchemy-utils
-sqlalchemy-utils==0.36.8
+sqlalchemy-utils==0.37.8
     # via
     #   apache-superset
     #   flask-appbuilder

requirements/development.in

@@ -18,7 +18,7 @@
 -r base.in
 flask-cors>=2.0.0
 mysqlclient==1.4.2.post1
-pillow>=7.0.0,<8.0.0
+pillow>=8.3.1,<9
 pydruid>=0.6.1,<0.7
 pyhive[hive]>=0.6.1
 psycopg2-binary==2.8.5

requirements/development.txt

@@ -1,4 +1,4 @@
-# SHA1:e4f3ea65026a8aec3735d6d9977f89fef4a1a4f9
+# SHA1:dbd3e93a11a36fc6b18d6194ac96ba29bd0ad2a8
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -40,7 +40,7 @@ mysqlclient==1.4.2.post1
     # via -r requirements/development.in
 openpyxl==3.0.7
     # via tabulator
-pillow==7.2.0
+pillow==8.3.1
     # via -r requirements/development.in
 progress==1.6
     # via -r requirements/development.in

setup.py

@@ -106,7 +106,7 @@ setup(
         "simplejson>=3.15.0",
         "slackclient==2.5.0",  # PINNED! slack changes file upload api in the future versions
         "sqlalchemy>=1.3.16, <1.4, !=1.3.21",
-        "sqlalchemy-utils>=0.36.6, <0.37",
+        "sqlalchemy-utils>=0.37.8, <0.38",
         "sqlparse==0.3.0",  # PINNED! see https://github.com/andialbrecht/sqlparse/issues/562
         "tabulate==0.8.9",
         "typing-extensions>=3.10, <4",  # needed to support Literal (3.8) and TypeGuard (3.10)
@@ -169,5 +169,6 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
 )

superset/utils/memoized.py

@@ -64,7 +64,9 @@ class _memoized:
         if not self.is_method:
             self.is_method = True
         # Support instance methods.
-        return functools.partial(self.__call__, obj)
+        func = functools.partial(self.__call__, obj)
+        func.__func__ = self.func  # type: ignore
+        return func
 
 
 def memoized(