diff --git a/.fossa.yml b/.fossa.yml index 04ec17845c..4273d9f4eb 100755 --- a/.fossa.yml +++ b/.fossa.yml @@ -32,7 +32,11 @@ analyze: type: pip target: docs path: docs + options: + requirements: ./requirements/documentation.txt - name: . type: pip target: . path: . + options: + requirements: ./requirements/testing.txt diff --git a/.github/workflows/bashlib.sh b/.github/workflows/bashlib.sh index 13d000d6c0..baabb17832 100644 --- a/.github/workflows/bashlib.sh +++ b/.github/workflows/bashlib.sh @@ -33,24 +33,21 @@ say() { # default command to run when the `run` input is empty default-setup-command() { - pip-install + apt-get-install + pip-upgrade } -# install python dependencies -pip-install() { - cd "$GITHUB_WORKSPACE" +apt-get-install() { + say "::group::apt-get install dependencies" + sudo apt-get update && sudo apt-get install --yes \ + libsasl2-dev + say "::endgroup::" +} - # Pip cache saves at most about 20s on a good day - # cache-restore pip - - say "::group::Install Python pacakges" +pip-upgrade() { + say "::group::Upgrade pip" pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt - pip install -e ".[postgres,mysql]" say "::endgroup::" - - # cache-save pip } # prepare (lint and build) frontend code @@ -123,6 +120,7 @@ testdata() { say "::group::Load test data" # must specify PYTHONPATH to make `tests.superset_test_config` importable export PYTHONPATH="$GITHUB_WORKSPACE" + pip install -e . superset db upgrade superset load_test_users superset load_examples --load-test-data diff --git a/.github/workflows/caches.js b/.github/workflows/caches.js index d2591c33c4..39d0ad2e6a 100644 --- a/.github/workflows/caches.js +++ b/.github/workflows/caches.js @@ -36,7 +36,7 @@ const assetsConfig = { module.exports = { pip: { path: [`${homeDirectory}/.cache/pip`], - hashFiles: [`${workspaceDirectory}/requirements*.txt`], + hashFiles: [`${workspaceDirectory}/requirements/*.txt`], }, npm: { path: [`${homeDirectory}/.npm`], diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 3c56903197..ad0aa63d9d 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -33,7 +33,7 @@ jobs: EOF - if [[ "${FILES}" =~ (.*package*\.json|requirements.*\.txt|setup\.py) ]]; then + if [[ "${FILES}" =~ (.*package*\.json|requirements/*\.txt|setup\.py) ]]; then echo "Detected dependency changes... running fossa check" ./scripts/fossa.sh diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 7e49cbba55..62156892c0 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -37,15 +37,18 @@ jobs: uses: actions/setup-python@v2.1.1 with: python-version: '3.6' - - name: Install dependencies uses: apache-superset/cached-dependencies@b90713b with: run: | - npm-install && build-instrumented-assets - pip-install && setup-postgres && testdata + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-postgres + testdata + npm-install + build-instrumented-assets cypress-install - - name: Run Cypress uses: apache-superset/cached-dependencies@b90713b env: diff --git a/.github/workflows/superset-python.yml b/.github/workflows/superset-python.yml index 72de728902..4b0c1a4af9 100644 --- a/.github/workflows/superset-python.yml +++ b/.github/workflows/superset-python.yml @@ -8,8 +8,6 @@ jobs: strategy: matrix: python-version: [3.6] - env: - PYTHON_LINT_TARGET: setup.py superset tests steps: - name: Checkout code uses: actions/checkout@v2 @@ -19,16 +17,37 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies uses: apache-superset/cached-dependencies@b90713b - - name: black - run: black --check $(echo $PYTHON_LINT_TARGET) - - name: mypy - run: mypy $(echo $PYTHON_LINT_TARGET) - - name: isort - run: isort --check-only --recursive $(echo $PYTHON_LINT_TARGET) + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt - name: pylint # `-j 0` run Pylint in parallel run: pylint -j 0 superset + pre-commit: + runs-on: ubuntu-18.04 + strategy: + matrix: + python-version: [3.6] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2.1.1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + uses: apache-superset/cached-dependencies@b90713b + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/integration.txt + - name: pre-commit + run: pre-commit run --all-files + docs: runs-on: ubuntu-18.04 strategy: @@ -45,8 +64,9 @@ jobs: uses: apache-superset/cached-dependencies@b90713b with: run: | - pip-install - pip install -r docs/requirements.txt + apt-get-install + pip-upgrade + pip install -r requirements/documentation.txt - name: Build documentation run: sphinx-build -b html docs _build/html -W @@ -66,8 +86,9 @@ jobs: uses: apache-superset/cached-dependencies@b90713b with: run: | - pip-install - pip install -r docs/requirements.txt + apt-get-install + pip-upgrade + pip install -r requirements/documentation.txt - name: Test babel extraction run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct @@ -107,7 +128,9 @@ jobs: uses: apache-superset/cached-dependencies@b90713b with: run: | - pip-install + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt setup-postgres - name: Run celery run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & @@ -151,7 +174,9 @@ jobs: uses: apache-superset/cached-dependencies@b90713b with: run: | - pip-install + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt setup-mysql - name: Run celery run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & @@ -188,7 +213,9 @@ jobs: uses: apache-superset/cached-dependencies@b90713b with: run: | - pip-install + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt mkdir ${{ github.workspace }}/.temp - name: Run celery run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 39b74e05b0..5edd6d47b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,29 +15,35 @@ # limitations under the License. # repos: -- repo: https://github.com/ambv/black - rev: 19.10b0 - hooks: - - id: black - language_version: python3 - -- repo: https://github.com/asottile/seed-isort-config + - repo: https://github.com/asottile/seed-isort-config rev: v1.9.3 hooks: - - id: seed-isort-config - -- repo: https://github.com/pre-commit/mirrors-isort + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort rev: v4.3.21 hooks: - - id: isort - -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 + - id: isort + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.770 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-docstring-first - - id: check-added-large-files - - id: check-yaml + - id: mypy + - repo: https://github.com/peterdemin/pip-compile-multi + rev: v1.5.8 + hooks: + - id: pip-compile-multi-verify + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: check-added-large-files + - id: check-docstring-first + - id: check-yaml exclude: ^helm/superset/templates/ - - id: debug-statements + - id: debug-statements + - id: end-of-file-fixer + - id: trailing-whitespace + args: ["--markdown-linebreak-ext=md"] + - repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black + language_version: python3 diff --git a/.pylintrc b/.pylintrc index e887176d65..e5a47b3381 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,7 +25,7 @@ # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS +ignore=CVS,migrations # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. diff --git a/.rat-excludes b/.rat-excludes index 8cd9d573dd..abe15af29e 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -1,3 +1,4 @@ +# Note: these patterns are applied to single files or directories, not full paths .gitignore .gitattributes .gitkeep @@ -40,7 +41,7 @@ _static/* .buildinfo searchindex.js # auto generated -requirements.txt +requirements/* # vendorized vendor/* # github configuration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a15cf48f3..e860092abd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -307,7 +307,7 @@ Finally, to make changes to the rst files and build the docs using Sphinx, you'll need to install a handful of dependencies from the repo you cloned: ```bash -pip install -r docs/requirements.txt +pip install -r requirements/documentation.txt ``` To get the feel for how to edit and build the docs, let's edit a file, build @@ -375,7 +375,7 @@ For example, the image referenced above actually lives in `superset-frontend/ima Generate the API documentation with: ```bash -pip install -r docs/requirements.txt +pip install -r requirements/documentation.txt python setup.py build_sphinx ``` @@ -399,8 +399,7 @@ virtualenv -p python3 venv # setup a python3.6 virtualenv source venv/bin/activate # Install external dependencies -pip install -r requirements.txt -pip install -r requirements-dev.txt +pip install -r requirements/local.txt # Install Superset in editable (development) mode pip install -e . @@ -553,7 +552,7 @@ in combined feature flags of `{ 'FOO': True, 'BAR': True, 'BAZ': True }`. Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following: ```bash -pip3 install -r requirements-dev.txt +pip3 install -r requirements/integration.txt pre-commit install ``` diff --git a/Dockerfile b/Dockerfile index 578f984c8f..b4b6a46c78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,9 +31,9 @@ RUN mkdir /app \ # First, we just wanna install requirements, which will allow us to utilize the cache # in order to only build if and only if requirements change -COPY ./requirements.txt /app/ +COPY ./requirements/*.txt /app/ RUN cd /app \ - && pip install --no-cache -r requirements.txt + && pip install --no-cache -r requirements/local.txt ###################################################################### @@ -114,14 +114,13 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] ###################################################################### FROM lean AS dev -COPY ./requirements* ./docker/requirements* /app/ +COPY ./requirements/*.txt ./docker/requirements/ /app/ USER root # Cache everything for dev purposes... RUN cd /app \ && pip install --ignore-installed -e . \ - && pip install --ignore-installed -r requirements.txt \ - && pip install --ignore-installed -r requirements-dev.txt \ + && pip install --ignore-installed -r requirements/local.txt \ && pip install --ignore-installed -r requirements-extra.txt \ && pip install --ignore-installed -r requirements-local.txt || true USER superset diff --git a/Dockerfile-dev b/Dockerfile-dev index f54f033d7c..f96b79fce7 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -20,12 +20,12 @@ ###################################################################### FROM preset/superset:dev -COPY ./requirements* ./docker/requirements* /app/ +COPY ./requirements/*.txt ./docker/requirements* /app/ USER root RUN cd /app \ && pip install -e . \ - && pip install --no-cache -r requirements.txt -r requirements-dev.txt \ + && pip install --no-cache -r requirements/local.txt \ && pip install --no-cache -r requirements-extra.txt \ && pip install --no-cache -r requirements-local.txt || true USER superset diff --git a/RELEASING/Dockerfile.make_docs b/RELEASING/Dockerfile.make_docs index 829fa75d8f..6ea0c68976 100644 --- a/RELEASING/Dockerfile.make_docs +++ b/RELEASING/Dockerfile.make_docs @@ -20,8 +20,7 @@ ARG VERSION RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/incubator-superset.git /superset WORKDIR /superset # install doc dependencies -RUN pip install -r requirements.txt \ - && pip install -r docs/requirements.txt +RUN pip install -r requirements/documentation.txt # build the docs RUN python setup.py build_sphinx diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 7270de3b96..35dd108b82 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -86,8 +86,8 @@ SQLLAB_CTAS_NO_LIMIT = True # the PYTHONPATH) in order to allow for local settings to be overridden # try: - from superset_config_docker import * # noqa import superset_config_docker + from superset_config_docker import * # noqa logger.info( f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]" diff --git a/docs/conf.py b/docs/conf.py index 8ae0e1f596..cefcc40194 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,6 +29,7 @@ import os import shlex import sys +from typing import Any, Dict import sphinx_rtd_theme @@ -130,7 +131,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = {} +html_theme_options: Dict[str, Any] = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] @@ -221,7 +222,7 @@ htmlhelp_basename = "supersetdoc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = { +latex_elements: Dict[str, Any] = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index da33b8ae81..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,104 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements.txt setup.py -# -aiohttp==3.6.2 # via slackclient -alembic==1.4.2 # via flask-migrate -amqp==2.6.0 # via kombu -apispec[yaml]==3.3.1 # via flask-appbuilder -async-timeout==3.0.1 # via aiohttp -attrs==19.3.0 # via aiohttp, jsonschema -babel==2.8.0 # via flask-babel -backoff==1.10.0 # via apache-superset (setup.py) -billiard==3.6.3.0 # via celery -bleach==3.1.5 # via apache-superset (setup.py) -brotli==1.0.7 # via flask-compress -cachelib==0.1.1 # via apache-superset (setup.py) -celery==4.4.6 # via apache-superset (setup.py) -cffi==1.14.0 # via cryptography -chardet==3.0.4 # via aiohttp -click==7.1.2 # via apache-superset (setup.py), flask, flask-appbuilder -colorama==0.4.3 # via apache-superset (setup.py), flask-appbuilder -contextlib2==0.6.0.post1 # via apache-superset (setup.py) -croniter==0.3.31 # via apache-superset (setup.py) -cryptography==2.9.2 # via apache-superset (setup.py) -dataclasses==0.6 # via apache-superset (setup.py) -decorator==4.4.2 # via retry -defusedxml==0.6.0 # via python3-openid -dnspython==1.16.0 # via email-validator -email-validator==1.1.0 # via flask-appbuilder -flask-appbuilder==3.0.1 # via apache-superset (setup.py) -flask-babel==1.0.0 # via flask-appbuilder -flask-caching==1.8.0 # via apache-superset (setup.py) -flask-compress==1.5.0 # via apache-superset (setup.py) -flask-jwt-extended==3.24.1 # via flask-appbuilder -flask-login==0.4.1 # via flask-appbuilder -flask-migrate==2.5.3 # via apache-superset (setup.py) -flask-openid==1.2.5 # via flask-appbuilder -flask-sqlalchemy==2.4.1 # via flask-appbuilder, flask-migrate -flask-talisman==0.7.0 # via apache-superset (setup.py) -flask-wtf==0.14.3 # via apache-superset (setup.py), flask-appbuilder -flask==1.1.2 # via apache-superset (setup.py), flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf -future==0.18.2 # via celery -geographiclib==1.50 # via geopy -geopy==1.22.0 # via apache-superset (setup.py) -gunicorn==20.0.4 # via apache-superset (setup.py) -humanize==2.4.0 # via apache-superset (setup.py) -idna-ssl==1.1.0 # via aiohttp -idna==2.9 # via email-validator, idna-ssl, yarl -importlib-metadata==1.6.0 # via jsonschema, kombu, markdown -isodate==0.6.0 # via apache-superset (setup.py) -itsdangerous==1.1.0 # via flask, flask-wtf -jinja2==2.11.2 # via flask, flask-babel -jsonschema==3.2.0 # via flask-appbuilder -kombu==4.6.11 # via celery -mako==1.1.2 # via alembic -markdown==3.2.2 # via apache-superset (setup.py) -markupsafe==1.1.1 # via jinja2, mako, wtforms -marshmallow-enum==1.5.1 # via flask-appbuilder -marshmallow-sqlalchemy==0.23.0 # via flask-appbuilder -marshmallow==3.6.1 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy -msgpack==1.0.0 # via apache-superset (setup.py) -multidict==4.7.6 # via aiohttp, yarl -numpy==1.18.4 # via pandas, pyarrow -packaging==20.3 # via bleach -pandas==1.0.3 # via apache-superset (setup.py) -parsedatetime==2.5 # via apache-superset (setup.py) -pathlib2==2.3.5 # via apache-superset (setup.py) -polyline==1.4.0 # via apache-superset (setup.py) -prison==0.1.3 # via flask-appbuilder -py==1.8.1 # via retry -pyarrow==0.17.0 # via apache-superset (setup.py) -pycparser==2.20 # via cffi -pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended -pyparsing==2.4.7 # via packaging -pyrsistent==0.16.0 # via jsonschema -python-dateutil==2.8.1 # via alembic, apache-superset (setup.py), croniter, flask-appbuilder, pandas -python-dotenv==0.13.0 # via apache-superset (setup.py) -python-editor==1.0.4 # via alembic -python-geohash==0.8.5 # via apache-superset (setup.py) -python3-openid==3.1.0 # via flask-openid -pytz==2020.1 # via babel, celery, flask-babel, pandas -pyyaml==5.3.1 # via apache-superset (setup.py), apispec -retry==0.9.2 # via apache-superset (setup.py) -selenium==3.141.0 # via apache-superset (setup.py) -simplejson==3.17.0 # via apache-superset (setup.py) -six==1.14.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json -slackclient==2.5.0 # via apache-superset (setup.py) -sqlalchemy-utils==0.36.6 # via apache-superset (setup.py), flask-appbuilder -sqlalchemy==1.3.16 # via alembic, apache-superset (setup.py), flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -sqlparse==0.3.0 # via apache-superset (setup.py) -typing-extensions==3.7.4.2 # via aiohttp -urllib3==1.25.9 # via selenium -vine==1.3.0 # via amqp, celery -webencodings==0.5.1 # via bleach -werkzeug==1.0.1 # via flask, flask-jwt-extended -wtforms-json==0.3.3 # via apache-superset (setup.py) -wtforms==2.3.1 # via flask-wtf, wtforms-json -yarl==1.4.2 # via aiohttp -zipp==3.1.0 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/superset/bin/superset b/requirements/base.in old mode 100755 new mode 100644 similarity index 88% rename from superset/bin/superset rename to requirements/base.in index e7b8de6846..e65985277d --- a/superset/bin/superset +++ b/requirements/base.in @@ -1,4 +1,4 @@ -#!/usr/bin/env python +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,7 +15,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from superset.cli import superset - -if __name__ == "__main__": - superset() +-e file:.[cors,druid,hive,mysql,postgres,thumbnails] diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 0000000000..d4f82d58bf --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,135 @@ +# SHA1:3b8e2a2d526726ac7ff836060bf89101ff52c1c8 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-e file:. # via -r requirements/base.in +aiohttp==3.6.2 # via slackclient +alembic==1.4.2 # via flask-migrate +amqp==2.6.1 # via kombu +apispec[yaml]==3.3.1 # via flask-appbuilder +async-timeout==3.0.1 # via aiohttp +attrs==19.3.0 # via aiohttp, jsonschema +babel==2.8.0 # via flask-babel +backoff==1.10.0 # via apache-superset +billiard==3.6.3.0 # via celery +bleach==3.1.5 # via apache-superset +boto3==1.14.34 # via tabulator +botocore==1.17.34 # via boto3, s3transfer +brotli==1.0.7 # via flask-compress +cached-property==1.5.1 # via tableschema +cachelib==0.1.1 # via apache-superset +celery==4.4.7 # via apache-superset +certifi==2020.6.20 # via requests +cffi==1.14.1 # via cryptography +chardet==3.0.4 # via aiohttp, requests, tabulator +click==7.1.2 # via apache-superset, flask, flask-appbuilder, tableschema, tabulator +colorama==0.4.3 # via apache-superset, flask-appbuilder +contextlib2==0.6.0.post1 # via apache-superset +croniter==0.3.34 # via apache-superset +cryptography==3.0 # via apache-superset +dataclasses==0.6 # via apache-superset +decorator==4.4.2 # via retry +defusedxml==0.6.0 # via python3-openid +dnspython==2.0.0 # via email-validator +docutils==0.15.2 # via botocore +email-validator==1.1.1 # via flask-appbuilder +et-xmlfile==1.0.1 # via openpyxl +flask-appbuilder==3.0.1 # via apache-superset +flask-babel==1.0.0 # via flask-appbuilder +flask-caching==1.9.0 # via apache-superset +flask-compress==1.5.0 # via apache-superset +flask-cors==3.0.8 # via apache-superset +flask-jwt-extended==3.24.1 # via flask-appbuilder +flask-login==0.4.1 # via flask-appbuilder +flask-migrate==2.5.3 # via apache-superset +flask-openid==1.2.5 # via flask-appbuilder +flask-sqlalchemy==2.4.4 # via flask-appbuilder, flask-migrate +flask-talisman==0.7.0 # via apache-superset +flask-wtf==0.14.3 # via apache-superset, flask-appbuilder +flask==1.1.2 # via apache-superset, flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-cors, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf +future==0.18.2 # via pyhive +geographiclib==1.50 # via geopy +geopy==2.0.0 # via apache-superset +gunicorn==20.0.4 # via apache-superset +humanize==2.5.0 # via apache-superset +idna-ssl==1.1.0 # via aiohttp +idna==2.10 # via email-validator, idna-ssl, requests, yarl +ijson==3.1.post0 # via tabulator +importlib-metadata==1.7.0 # via jsonschema, kombu, markdown +isodate==0.6.0 # via apache-superset, tableschema +itsdangerous==1.1.0 # via flask, flask-wtf +jdcal==1.4.1 # via openpyxl +jinja2==2.11.2 # via flask, flask-babel +jmespath==0.10.0 # via boto3, botocore +jsonlines==1.2.0 # via tabulator +jsonschema==3.2.0 # via flask-appbuilder, tableschema +kombu==4.6.11 # via celery +linear-tsv==1.1.0 # via tabulator +mako==1.1.3 # via alembic +markdown==3.2.2 # via apache-superset +markupsafe==1.1.1 # via jinja2, mako, wtforms +marshmallow-enum==1.5.1 # via flask-appbuilder +marshmallow-sqlalchemy==0.23.1 # via flask-appbuilder +marshmallow==3.7.1 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy +msgpack==1.0.0 # via apache-superset +multidict==4.7.6 # via aiohttp, yarl +mysqlclient==1.4.2.post1 # via apache-superset +natsort==7.0.1 # via croniter +numpy==1.19.1 # via pandas, pyarrow +openpyxl==3.0.4 # via tabulator +packaging==20.4 # via bleach +pandas==1.0.5 # via apache-superset +parsedatetime==2.6 # via apache-superset +pathlib2==2.3.5 # via apache-superset +pillow==7.2.0 # via apache-superset +polyline==1.4.0 # via apache-superset +prison==0.1.3 # via flask-appbuilder +psycopg2-binary==2.8.5 # via apache-superset +py==1.9.0 # via retry +pyarrow==0.17.1 # via apache-superset +pycparser==2.20 # via cffi +pydruid==0.6.1 # via apache-superset +pyhive[hive]==0.6.2 # via apache-superset +pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended +pyparsing==2.4.7 # via packaging +pyrsistent==0.16.0 # via jsonschema +python-dateutil==2.8.1 # via alembic, apache-superset, botocore, croniter, flask-appbuilder, pandas, pyhive, tableschema +python-dotenv==0.14.0 # via apache-superset +python-editor==1.0.4 # via alembic +python-geohash==0.8.5 # via apache-superset +python3-openid==3.2.0 # via flask-openid +pytz==2020.1 # via babel, celery, flask-babel, pandas +pyyaml==5.3.1 # via apache-superset, apispec +requests==2.24.0 # via pydruid, tableschema, tabulator +retry==0.9.2 # via apache-superset +rfc3986==1.4.0 # via tableschema +s3transfer==0.3.3 # via boto3 +sasl==0.2.1 # via pyhive, thrift-sasl +selenium==3.141.0 # via apache-superset +simplejson==3.17.2 # via apache-superset +six==1.15.0 # via bleach, cryptography, flask-cors, flask-jwt-extended, flask-talisman, isodate, jsonlines, jsonschema, linear-tsv, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sasl, sqlalchemy-utils, tableschema, tabulator, thrift, thrift-sasl, wtforms-json +slackclient==2.5.0 # via apache-superset +sqlalchemy-utils==0.36.8 # via apache-superset, flask-appbuilder +sqlalchemy==1.3.18 # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils, tabulator +sqlparse==0.3.0 # via apache-superset +tableschema==1.19.2 # via apache-superset +tabulator==1.52.3 # via tableschema +thrift-sasl==0.4.2 # via pyhive +thrift==0.13.0 # via apache-superset, pyhive, thrift-sasl +typing-extensions==3.7.4.2 # via aiohttp, yarl +unicodecsv==0.14.1 # via tableschema, tabulator +urllib3==1.25.10 # via botocore, requests, selenium +vine==1.3.0 # via amqp, celery +webencodings==0.5.1 # via bleach +werkzeug==1.0.1 # via flask, flask-jwt-extended +wtforms-json==0.3.3 # via apache-superset +wtforms==2.3.3 # via flask-wtf, wtforms-json +xlrd==1.2.0 # via tabulator +yarl==1.5.1 # via aiohttp +zipp==3.1.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/docs/requirements.txt b/requirements/documentation.in similarity index 95% rename from docs/requirements.txt rename to requirements/documentation.in index a8a226f79a..4b9eed06af 100644 --- a/docs/requirements.txt +++ b/requirements/documentation.in @@ -14,5 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -sphinx==3.0.1 -sphinx-rtd-theme==0.4.3 +-r base.in +sphinx +sphinx-rtd-theme diff --git a/requirements/documentation.txt b/requirements/documentation.txt new file mode 100644 index 0000000000..e963a8659a --- /dev/null +++ b/requirements/documentation.txt @@ -0,0 +1,24 @@ +# SHA1:2cf3a7081bf9189afa931880771c21608989698b +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +-e file:. # via -r requirements/base.in +alabaster==0.7.12 # via sphinx +imagesize==1.2.0 # via sphinx +pygments==2.6.1 # via sphinx +snowballstemmer==2.0.0 # via sphinx +sphinx-rtd-theme==0.5.0 # via -r requirements/documentation.in +sphinx==3.1.2 # via -r requirements/documentation.in, sphinx-rtd-theme +sphinxcontrib-applehelp==1.0.2 # via sphinx +sphinxcontrib-devhelp==1.0.2 # via sphinx +sphinxcontrib-htmlhelp==1.0.3 # via sphinx +sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-qthelp==1.0.3 # via sphinx +sphinxcontrib-serializinghtml==1.1.4 # via sphinx + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements-dev.txt b/requirements/integration.in similarity index 62% rename from requirements-dev.txt rename to requirements/integration.in index d68a17b010..1aaee9255f 100644 --- a/requirements-dev.txt +++ b/requirements/integration.in @@ -14,28 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -black==19.10b0 -coverage==4.5.3 -flask-cors==3.0.7 -flask-testing==0.8.0 -ipdb==0.12 -isort==4.3.21 -mypy==0.770 -openapi-spec-validator==0.2.8 -pytest==5.4.3 -pytest-cov==2.10.0 -parameterized==0.7.4 -pip-tools==5.1.2 -pre-commit==1.17.0 -psycopg2-binary==2.8.5 -pycodestyle==2.5.0 -pydruid==0.6.1 -pyhive==0.6.2 -pylint==2.5.3 -redis==3.5.1 -requests==2.23.0 -statsd==3.3.0 -tox==3.11.1 -pillow==7.0.0 -openpyxl==3.0.3 # Pandas use openpyxl to write excel format(using in unittes) -xlrd==1.2.0 +pip-compile-multi!=1.5.9 +pre-commit +tox diff --git a/requirements/integration.txt b/requirements/integration.txt new file mode 100644 index 0000000000..019d2dc795 --- /dev/null +++ b/requirements/integration.txt @@ -0,0 +1,33 @@ +# SHA1:2e1bdbb387f30da2c99c8ca9cbf7ce42efba84cf +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +appdirs==1.4.4 # via virtualenv +cfgv==3.2.0 # via pre-commit +click==7.1.2 # via pip-compile-multi, pip-tools +distlib==0.3.1 # via virtualenv +filelock==3.0.12 # via tox, virtualenv +identify==1.4.25 # via pre-commit +importlib-metadata==1.7.0 # via pluggy, pre-commit, tox, virtualenv +importlib-resources==3.0.0 # via pre-commit, virtualenv +nodeenv==1.4.0 # via pre-commit +packaging==20.4 # via tox +pip-compile-multi==1.5.8 # via -r requirements/integration.in +pip-tools==5.3.1 # via pip-compile-multi +pluggy==0.13.1 # via tox +pre-commit==2.6.0 # via -r requirements/integration.in +py==1.9.0 # via tox +pyparsing==2.4.7 # via packaging +pyyaml==5.3.1 # via pre-commit +six==1.15.0 # via packaging, pip-tools, tox, virtualenv +toml==0.10.1 # via pre-commit, tox +toposort==1.5 # via pip-compile-multi +tox==3.18.1 # via -r requirements/integration.in +virtualenv==20.0.29 # via pre-commit, tox +zipp==3.1.0 # via importlib-metadata, importlib-resources + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements/local.in b/requirements/local.in new file mode 100644 index 0000000000..8a9ad706f1 --- /dev/null +++ b/requirements/local.in @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +-r base.in diff --git a/requirements/local.txt b/requirements/local.txt new file mode 100644 index 0000000000..b88dd1b325 --- /dev/null +++ b/requirements/local.txt @@ -0,0 +1,12 @@ +# SHA1:37fdd6cd1e7782cad40e1c59a0b7b6e1d9f73332 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +-e file:. # via -r requirements/base.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/testing.in b/requirements/testing.in new file mode 100644 index 0000000000..1d18524fe3 --- /dev/null +++ b/requirements/testing.in @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +-r base.in +-r integration.in +flask-testing +openapi-spec-validator +openpyxl +parameterized +pylint +pytest +pytest-cov +redis +statsd diff --git a/requirements/testing.txt b/requirements/testing.txt new file mode 100644 index 0000000000..9515dd7bff --- /dev/null +++ b/requirements/testing.txt @@ -0,0 +1,31 @@ +# SHA1:785ae7ffcde3cee8ebcc0a839cdb8e61e693d329 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +-r integration.txt +-e file:. # via -r requirements/base.in +astroid==2.4.2 # via pylint +coverage==5.2.1 # via pytest-cov +flask-testing==0.8.0 # via -r requirements/testing.in +iniconfig==1.0.1 # via pytest +isort==4.3.21 # via pylint +lazy-object-proxy==1.4.3 # via astroid +mccabe==0.6.1 # via pylint +more-itertools==8.4.0 # via pytest +openapi-spec-validator==0.2.9 # via -r requirements/testing.in +parameterized==0.7.4 # via -r requirements/testing.in +pylint==2.5.3 # via -r requirements/testing.in +pytest-cov==2.10.0 # via -r requirements/testing.in +pytest==6.0.1 # via -r requirements/testing.in, pytest-cov +redis==3.5.3 # via -r requirements/testing.in +statsd==3.3.0 # via -r requirements/testing.in +typed-ast==1.4.1 # via astroid +wrapt==1.12.1 # via astroid + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/scripts/permissions_cleanup.py b/scripts/permissions_cleanup.py index c37d124456..3656aaa43d 100644 --- a/scripts/permissions_cleanup.py +++ b/scripts/permissions_cleanup.py @@ -16,12 +16,14 @@ # under the License. from collections import defaultdict -from superset import sm +from superset import security_manager def cleanup_permissions(): # 1. Clean up duplicates. - pvms = sm.get_session.query(sm.permissionview_model).all() + pvms = security_manager.get_session.query( + security_manager.permissionview_model + ).all() print("# of permission view menues is: {}".format(len(pvms))) pvms_dict = defaultdict(list) for pvm in pvms: @@ -34,34 +36,42 @@ def cleanup_permissions(): roles = set(first_prm.role) for pvm in pvm_list[1:]: roles = roles.union(pvm.role) - sm.get_session.delete(pvm) + security_manager.get_session.delete(pvm) first_prm.roles = list(roles) - sm.get_session.commit() + security_manager.get_session.commit() - pvms = sm.get_session.query(sm.permissionview_model).all() - print("STage 1: # of permission view menues is: {}".format(len(pvms))) + pvms = security_manager.get_session.query( + security_manager.permissionview_model + ).all() + print("Stage 1: # of permission view menues is: {}".format(len(pvms))) # 2. Clean up None permissions or view menues - pvms = sm.get_session.query(sm.permissionview_model).all() + pvms = security_manager.get_session.query( + security_manager.permissionview_model + ).all() for pvm in pvms: if not (pvm.view_menu and pvm.permission): - sm.get_session.delete(pvm) - sm.get_session.commit() + security_manager.get_session.delete(pvm) + security_manager.get_session.commit() - pvms = sm.get_session.query(sm.permissionview_model).all() + pvms = security_manager.get_session.query( + security_manager.permissionview_model + ).all() print("Stage 2: # of permission view menues is: {}".format(len(pvms))) # 3. Delete empty permission view menues from roles - roles = sm.get_session.query(sm.role_model).all() + roles = security_manager.get_session.query(security_manager.role_model).all() for role in roles: role.permissions = [p for p in role.permissions if p] - sm.get_session.commit() + security_manager.get_session.commit() # 4. Delete empty roles from permission view menues - pvms = sm.get_session.query(sm.permissionview_model).all() + pvms = security_manager.get_session.query( + security_manager.permissionview_model + ).all() for pvm in pvms: pvm.role = [r for r in pvm.role if r] - sm.get_session.commit() + security_manager.get_session.commit() cleanup_permissions() diff --git a/setup.cfg b/setup.cfg index c126a4a0b8..18a07194e2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,3 +52,6 @@ warn_unused_ignores = true check_untyped_defs = true disallow_untyped_calls = true disallow_untyped_defs = true + +[mypy-superset.migrations.versions.*] +ignore_errors = true diff --git a/setup.py b/setup.py index 2a21920448..03e86d26f9 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup( packages=find_packages(), include_package_data=True, zip_safe=False, - scripts=["superset/bin/superset"], + entry_points={"console_scripts": ["superset=superset.cli:superset"]}, install_requires=[ "backoff>=1.8.0", "bleach>=3.0.2, <4.0.0", @@ -135,5 +135,4 @@ setup( "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", ], - tests_require=["flask-testing==0.8.0"], ) diff --git a/superset/app.py b/superset/app.py index ad330f3f8d..b64ca69c5b 100644 --- a/superset/app.py +++ b/superset/app.py @@ -124,63 +124,62 @@ class SupersetAppInitializer: # # pylint: disable=too-many-locals # pylint: disable=too-many-statements + from superset.charts.api import ChartRestApi from superset.connectors.druid.views import ( - DruidDatasourceModelView, - DruidClusterModelView, - DruidMetricInlineView, - DruidColumnInlineView, Druid, + DruidClusterModelView, + DruidColumnInlineView, + DruidDatasourceModelView, + DruidMetricInlineView, ) + from superset.connectors.sqla.views import ( + RowLevelSecurityFiltersModelView, + SqlMetricInlineView, + TableColumnInlineView, + TableModelView, + ) + from superset.dashboards.api import DashboardRestApi + from superset.databases.api import DatabaseRestApi from superset.datasets.api import DatasetRestApi from superset.queries.api import QueryRestApi - from superset.connectors.sqla.views import ( - TableColumnInlineView, - SqlMetricInlineView, - TableModelView, - RowLevelSecurityFiltersModelView, - ) + from superset.views.access_requests import AccessRequestsModelView + from superset.views.alerts import AlertLogModelView, AlertModelView from superset.views.annotations import ( AnnotationLayerModelView, AnnotationModelView, ) from superset.views.api import Api + from superset.views.chart.views import SliceAsync, SliceModelView from superset.views.core import Superset - from superset.views.redirects import R - from superset.views.key_value import KV - from superset.views.access_requests import AccessRequestsModelView - from superset.views.css_templates import CssTemplateAsyncModelView - from superset.views.css_templates import CssTemplateModelView - from superset.charts.api import ChartRestApi - from superset.views.chart.views import SliceModelView, SliceAsync - from superset.dashboards.api import DashboardRestApi + from superset.views.css_templates import ( + CssTemplateAsyncModelView, + CssTemplateModelView, + ) from superset.views.dashboard.views import ( - DashboardModelView, Dashboard, + DashboardModelView, DashboardModelViewAsync, ) - from superset.databases.api import DatabaseRestApi from superset.views.database.views import ( - DatabaseView, CsvToDatabaseView, + DatabaseView, ExcelToDatabaseView, ) from superset.views.datasource import Datasource + from superset.views.key_value import KV from superset.views.log.api import LogRestApi from superset.views.log.views import LogModelView + from superset.views.redirects import R from superset.views.schedules import ( DashboardEmailScheduleView, SliceEmailScheduleView, ) - from superset.views.alerts import ( - AlertModelView, - AlertLogModelView, - ) from superset.views.sql_lab import ( - SavedQueryViewApi, SavedQueryView, - TabStateView, - TableSchemaView, + SavedQueryViewApi, SqlLab, + TableSchemaView, + TabStateView, ) from superset.views.tags import TagView @@ -541,8 +540,10 @@ class SupersetAppInitializer: # Doing local imports here as model importing causes a reference to # app.config to be invoked and we need the current_app to have been setup # - from superset.utils.url_map_converters import RegexConverter - from superset.utils.url_map_converters import ObjectTypeConverter + from superset.utils.url_map_converters import ( + ObjectTypeConverter, + RegexConverter, + ) self.flask_app.url_map.converters["regex"] = RegexConverter self.flask_app.url_map.converters["object_type"] = ObjectTypeConverter diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py index 96fc7b520a..34fa4d8c53 100644 --- a/superset/charts/schemas.py +++ b/superset/charts/schemas.py @@ -402,7 +402,7 @@ class ChartDataSelectOptionsSchema(ChartDataPostProcessingOperationOptionsSchema "referenced here.", example=["country", "gender", "age"], ) - exclude = fields.List( # type: ignore + exclude = fields.List( fields.String(), description="Columns to exclude from selection.", example=["my_temp_column"], diff --git a/superset/cli.py b/superset/cli.py index 1e47f1ea9a..ef9ee847fb 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -613,8 +613,8 @@ def sync_tags() -> None: def alert() -> None: """Run the alert scheduler loop""" # this command is just for testing purposes - from superset.tasks.schedules import schedule_window from superset.models.schedules import ScheduleType + from superset.tasks.schedules import schedule_window click.secho("Processing one alert loop", fg="green") schedule_window( diff --git a/superset/commands/exceptions.py b/superset/commands/exceptions.py index d470043bb3..f5086725fb 100644 --- a/superset/commands/exceptions.py +++ b/superset/commands/exceptions.py @@ -49,7 +49,7 @@ class CommandInvalidError(CommandException): def normalized_messages(self) -> Dict[Any, Any]: errors: Dict[Any, Any] = {} for exception in self._invalid_exceptions: - errors.update(exception.normalized_messages()) # type: ignore + errors.update(exception.normalized_messages()) return errors diff --git a/superset/config.py b/superset/config.py index da97469b67..cfef8c2af7 100644 --- a/superset/config.py +++ b/superset/config.py @@ -49,6 +49,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: from flask_appbuilder.security.sqla import models # pylint: disable=unused-import + from superset.models.core import Database # pylint: disable=unused-import # Realtime stats logger, a StatsD implementation exists @@ -918,8 +919,8 @@ if CONFIG_PATH_ENV_VAR in os.environ: raise elif importlib.util.find_spec("superset_config"): try: - from superset_config import * # pylint: disable=import-error,wildcard-import,unused-wildcard-import import superset_config # pylint: disable=import-error + from superset_config import * # type: ignore # pylint: disable=import-error,wildcard-import,unused-wildcard-import print(f"Loaded your LOCAL configuration at [{superset_config.__file__}]") except Exception: diff --git a/superset/connectors/connector_registry.py b/superset/connectors/connector_registry.py index 8b5998ff8b..7c47bc7b3f 100644 --- a/superset/connectors/connector_registry.py +++ b/superset/connectors/connector_registry.py @@ -25,8 +25,8 @@ if TYPE_CHECKING: # pylint: disable=unused-import from collections import OrderedDict - from superset.models.core import Database from superset.connectors.base.models import BaseDatasource + from superset.models.core import Database class ConnectorRegistry: diff --git a/superset/connectors/druid/models.py b/superset/connectors/druid/models.py index 4cdd78a3fd..0068f111ce 100644 --- a/superset/connectors/druid/models.py +++ b/superset/connectors/druid/models.py @@ -59,6 +59,7 @@ from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict from superset.utils import core as utils, import_datasource try: + import requests from pydruid.client import PyDruid from pydruid.utils.aggregators import count from pydruid.utils.dimensions import ( @@ -77,7 +78,6 @@ try: Quantile, Quantiles, ) - import requests except ImportError: pass diff --git a/superset/db_engine_specs/hive.py b/superset/db_engine_specs/hive.py index 4cafaaf2b2..206e1c7d35 100644 --- a/superset/db_engine_specs/hive.py +++ b/superset/db_engine_specs/hive.py @@ -89,13 +89,14 @@ class HiveEngineSpec(PrestoEngineSpec): @classmethod def patch(cls) -> None: from pyhive import hive # pylint: disable=no-name-in-module - from superset.db_engines import hive as patched_hive from TCLIService import ( constants as patched_constants, - ttypes as patched_ttypes, TCLIService as patched_TCLIService, + ttypes as patched_ttypes, ) + from superset.db_engines import hive as patched_hive + hive.TCLIService = patched_TCLIService hive.constants = patched_constants hive.ttypes = patched_ttypes diff --git a/superset/bin/__init__.py b/superset/migrations/versions/__init__.py similarity index 100% rename from superset/bin/__init__.py rename to superset/migrations/versions/__init__.py diff --git a/superset/viz.py b/superset/viz.py index 3ce98e4b8d..7d14bad29b 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -1374,8 +1374,8 @@ class MultiLineViz(NVD3Viz): def get_data(self, df: pd.DataFrame) -> VizData: fd = self.form_data # Late imports to avoid circular import issues - from superset.models.slice import Slice from superset import db + from superset.models.slice import Slice slice_ids1 = fd.get("line_charts") slices1 = db.session.query(Slice).filter(Slice.id.in_(slice_ids1)).all() @@ -2284,8 +2284,8 @@ class DeckGLMultiLayer(BaseViz): def get_data(self, df: pd.DataFrame) -> VizData: fd = self.form_data # Late imports to avoid circular import issues - from superset.models.slice import Slice from superset import db + from superset.models.slice import Slice slice_ids = fd.get("deck_slices") slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all() diff --git a/superset/viz_sip38.py b/superset/viz_sip38.py index aa7a4e1f77..4b8c6a59b3 100644 --- a/superset/viz_sip38.py +++ b/superset/viz_sip38.py @@ -1356,8 +1356,8 @@ class MultiLineViz(NVD3Viz): def get_data(self, df: pd.DataFrame) -> VizData: fd = self.form_data # Late imports to avoid circular import issues - from superset.models.slice import Slice from superset import db + from superset.models.slice import Slice slice_ids1 = fd.get("line_charts") slices1 = db.session.query(Slice).filter(Slice.id.in_(slice_ids1)).all() @@ -2174,8 +2174,8 @@ class DeckGLMultiLayer(BaseViz): def get_data(self, df: pd.DataFrame) -> VizData: fd = self.form_data # Late imports to avoid circular import issues - from superset.models.slice import Slice from superset import db + from superset.models.slice import Slice slice_ids = fd.get("deck_slices") slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all() diff --git a/tox.ini b/tox.ini index 9ff47a24fa..6e9f19d0dc 100644 --- a/tox.ini +++ b/tox.ini @@ -23,10 +23,7 @@ commands = # no args or tests/* can be passed as an argument to run all tests pytest --ignore=load_examples_test {posargs} deps = - -rrequirements.txt - -rrequirements-dev.txt - mysql: .[mysql] - postgres: .[postgres] + -rrequirements/testing.txt setenv = PYTHONPATH = {toxinidir} SUPERSET_CONFIG = tests.superset_test_config @@ -37,20 +34,13 @@ setenv = whitelist_externals = npm -[testenv:black] -commands = - black --check setup.py superset tests -deps = - -rrequirements-dev.txt - [testenv:cypress-dashboard] commands = npm install -g npm@'>=6.5.0' pip install -e {toxinidir}/ {toxinidir}/superset-frontend/cypress_build.sh dashboard deps = - -rrequirements.txt - -rrequirements-dev.txt + -rrequirements/testing.txt setenv = PYTHONPATH = {toxinidir} SUPERSET_CONFIG = tests.superset_test_config @@ -62,8 +52,7 @@ commands = pip install -e {toxinidir}/ {toxinidir}/superset-frontend/cypress_build.sh explore deps = - -rrequirements.txt - -rrequirements-dev.txt + -rrequirements/testing.txt setenv = PYTHONPATH = {toxinidir} SUPERSET_CONFIG = tests.superset_test_config @@ -75,8 +64,7 @@ commands = pip install -e {toxinidir}/ {toxinidir}/superset-frontend/cypress_build.sh sqllab deps = - -rrequirements.txt - -rrequirements-dev.txt + -rrequirements/testing.txt setenv = PYTHONPATH = {toxinidir} SUPERSET_CONFIG = tests.superset_test_config @@ -88,8 +76,7 @@ commands = pip install -e {toxinidir}/ {toxinidir}/superset-frontend/cypress_build.sh sqllab deps = - -rrequirements.txt - -rrequirements-dev.txt + -rrequirements/testing.txt setenv = PYTHONPATH = {toxinidir} SUPERSET_CONFIG = tests.superset_test_config_sqllab_backend_persist @@ -99,9 +86,7 @@ setenv = commands = sphinx-build -b html docs _build/html -W deps = - -rrequirements.txt - -rrequirements-dev.txt - -rdocs/requirements.txt + -rrequirements/documentation.txt [testenv:eslint] changedir = {toxinidir}/superset-frontend @@ -115,12 +100,6 @@ commands = deps = passenv = * -[testenv:isort] -commands = - isort --check-only --recursive setup.py superset tests -deps = - -rrequirements-dev.txt - [testenv:javascript] commands = npm install -g npm@'>=6.5.0' @@ -135,30 +114,27 @@ whitelist_externals = {toxinidir}/scripts/check_license.sh deps = -[testenv:mypy] +[testenv:pre-commit] commands = - mypy setup.py superset tests + pre-commit run --all-files deps = - -rrequirements-dev.txt + -rrequirements/integration.txt +skip_install = true [testenv:pylint] commands = pylint superset deps = - -rrequirements.txt - -rrequirements-dev.txt + -rrequirements/testing.txt [testenv:thumbnails] setenv = SUPERSET_CONFIG = tests.superset_test_config_thumbnails deps = - -rrequirements.txt - -rrequirements-dev.txt - .[postgres] + -rrequirements/testing.txt [tox] envlist = - black cypress-dashboard cypress-explore cypress-sqllab @@ -166,9 +142,8 @@ envlist = docs eslint fossa - isort javascript license-check - mypy + pre-commit pylint skipsdist = true