chore: Adding pip-compile-multi et al. (#10499)

* chore: Adding pip-compile-multi et al

* Specify requirements.txt path for fossa

* [ci] Fixing CI

Co-authored-by: John Bodley <john.bodley@airbnb.com>
Co-authored-by: Jesse Yang <jesse.yang@airbnb.com>
This commit is contained in:
John Bodley 2020-08-04 19:31:52 -07:00 committed by GitHub
parent 72ced53d2a
commit cda764fd4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 483 additions and 305 deletions

View File

@ -32,7 +32,11 @@ analyze:
type: pip
target: docs
path: docs
options:
requirements: ./requirements/documentation.txt
- name: .
type: pip
target: .
path: .
options:
requirements: ./requirements/testing.txt

View File

@ -33,24 +33,21 @@ say() {
# default command to run when the `run` input is empty
default-setup-command() {
pip-install
apt-get-install
pip-upgrade
}
# install python dependencies
pip-install() {
cd "$GITHUB_WORKSPACE"
apt-get-install() {
say "::group::apt-get install dependencies"
sudo apt-get update && sudo apt-get install --yes \
libsasl2-dev
say "::endgroup::"
}
# Pip cache saves at most about 20s on a good day
# cache-restore pip
say "::group::Install Python pacakges"
pip-upgrade() {
say "::group::Upgrade pip"
pip install --upgrade pip
pip install -r requirements.txt
pip install -r requirements-dev.txt
pip install -e ".[postgres,mysql]"
say "::endgroup::"
# cache-save pip
}
# prepare (lint and build) frontend code
@ -123,6 +120,7 @@ testdata() {
say "::group::Load test data"
# must specify PYTHONPATH to make `tests.superset_test_config` importable
export PYTHONPATH="$GITHUB_WORKSPACE"
pip install -e .
superset db upgrade
superset load_test_users
superset load_examples --load-test-data

View File

@ -36,7 +36,7 @@ const assetsConfig = {
module.exports = {
pip: {
path: [`${homeDirectory}/.cache/pip`],
hashFiles: [`${workspaceDirectory}/requirements*.txt`],
hashFiles: [`${workspaceDirectory}/requirements/*.txt`],
},
npm: {
path: [`${homeDirectory}/.npm`],

View File

@ -33,7 +33,7 @@ jobs:
EOF
if [[ "${FILES}" =~ (.*package*\.json|requirements.*\.txt|setup\.py) ]]; then
if [[ "${FILES}" =~ (.*package*\.json|requirements/*\.txt|setup\.py) ]]; then
echo "Detected dependency changes... running fossa check"
./scripts/fossa.sh

View File

@ -37,15 +37,18 @@ jobs:
uses: actions/setup-python@v2.1.1
with:
python-version: '3.6'
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
npm-install && build-instrumented-assets
pip-install && setup-postgres && testdata
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
testdata
npm-install
build-instrumented-assets
cypress-install
- name: Run Cypress
uses: apache-superset/cached-dependencies@b90713b
env:

View File

@ -8,8 +8,6 @@ jobs:
strategy:
matrix:
python-version: [3.6]
env:
PYTHON_LINT_TARGET: setup.py superset tests
steps:
- name: Checkout code
uses: actions/checkout@v2
@ -19,16 +17,37 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
- name: black
run: black --check $(echo $PYTHON_LINT_TARGET)
- name: mypy
run: mypy $(echo $PYTHON_LINT_TARGET)
- name: isort
run: isort --check-only --recursive $(echo $PYTHON_LINT_TARGET)
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
- name: pylint
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
pre-commit:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [3.6]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/integration.txt
- name: pre-commit
run: pre-commit run --all-files
docs:
runs-on: ubuntu-18.04
strategy:
@ -45,8 +64,9 @@ jobs:
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
pip install -r docs/requirements.txt
apt-get-install
pip-upgrade
pip install -r requirements/documentation.txt
- name: Build documentation
run: sphinx-build -b html docs _build/html -W
@ -66,8 +86,9 @@ jobs:
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
pip install -r docs/requirements.txt
apt-get-install
pip-upgrade
pip install -r requirements/documentation.txt
- name: Test babel extraction
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
@ -107,7 +128,9 @@ jobs:
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
@ -151,7 +174,9 @@ jobs:
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
@ -188,7 +213,9 @@ jobs:
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &

View File

@ -15,29 +15,35 @@
# limitations under the License.
#
repos:
- repo: https://github.com/ambv/black
rev: 19.10b0
hooks:
- id: black
language_version: python3
- repo: https://github.com/asottile/seed-isort-config
- repo: https://github.com/asottile/seed-isort-config
rev: v1.9.3
hooks:
- id: seed-isort-config
- repo: https://github.com/pre-commit/mirrors-isort
- id: seed-isort-config
- repo: https://github.com/pre-commit/mirrors-isort
rev: v4.3.21
hooks:
- id: isort
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.2.3
- id: isort
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.770
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-docstring-first
- id: check-added-large-files
- id: check-yaml
- id: mypy
- repo: https://github.com/peterdemin/pip-compile-multi
rev: v1.5.8
hooks:
- id: pip-compile-multi-verify
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.2.0
hooks:
- id: check-added-large-files
- id: check-docstring-first
- id: check-yaml
exclude: ^helm/superset/templates/
- id: debug-statements
- id: debug-statements
- id: end-of-file-fixer
- id: trailing-whitespace
args: ["--markdown-linebreak-ext=md"]
- repo: https://github.com/psf/black
rev: 19.10b0
hooks:
- id: black
language_version: python3

View File

@ -25,7 +25,7 @@
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
ignore=CVS,migrations
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.

View File

@ -1,3 +1,4 @@
# Note: these patterns are applied to single files or directories, not full paths
.gitignore
.gitattributes
.gitkeep
@ -40,7 +41,7 @@ _static/*
.buildinfo
searchindex.js
# auto generated
requirements.txt
requirements/*
# vendorized
vendor/*
# github configuration

View File

@ -307,7 +307,7 @@ Finally, to make changes to the rst files and build the docs using Sphinx,
you'll need to install a handful of dependencies from the repo you cloned:
```bash
pip install -r docs/requirements.txt
pip install -r requirements/documentation.txt
```
To get the feel for how to edit and build the docs, let's edit a file, build
@ -375,7 +375,7 @@ For example, the image referenced above actually lives in `superset-frontend/ima
Generate the API documentation with:
```bash
pip install -r docs/requirements.txt
pip install -r requirements/documentation.txt
python setup.py build_sphinx
```
@ -399,8 +399,7 @@ virtualenv -p python3 venv # setup a python3.6 virtualenv
source venv/bin/activate
# Install external dependencies
pip install -r requirements.txt
pip install -r requirements-dev.txt
pip install -r requirements/local.txt
# Install Superset in editable (development) mode
pip install -e .
@ -553,7 +552,7 @@ in combined feature flags of `{ 'FOO': True, 'BAR': True, 'BAZ': True }`.
Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following:
```bash
pip3 install -r requirements-dev.txt
pip3 install -r requirements/integration.txt
pre-commit install
```

View File

@ -31,9 +31,9 @@ RUN mkdir /app \
# First, we just wanna install requirements, which will allow us to utilize the cache
# in order to only build if and only if requirements change
COPY ./requirements.txt /app/
COPY ./requirements/*.txt /app/
RUN cd /app \
&& pip install --no-cache -r requirements.txt
&& pip install --no-cache -r requirements/local.txt
######################################################################
@ -114,14 +114,13 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
######################################################################
FROM lean AS dev
COPY ./requirements* ./docker/requirements* /app/
COPY ./requirements/*.txt ./docker/requirements/ /app/
USER root
# Cache everything for dev purposes...
RUN cd /app \
&& pip install --ignore-installed -e . \
&& pip install --ignore-installed -r requirements.txt \
&& pip install --ignore-installed -r requirements-dev.txt \
&& pip install --ignore-installed -r requirements/local.txt \
&& pip install --ignore-installed -r requirements-extra.txt \
&& pip install --ignore-installed -r requirements-local.txt || true
USER superset

View File

@ -20,12 +20,12 @@
######################################################################
FROM preset/superset:dev
COPY ./requirements* ./docker/requirements* /app/
COPY ./requirements/*.txt ./docker/requirements* /app/
USER root
RUN cd /app \
&& pip install -e . \
&& pip install --no-cache -r requirements.txt -r requirements-dev.txt \
&& pip install --no-cache -r requirements/local.txt \
&& pip install --no-cache -r requirements-extra.txt \
&& pip install --no-cache -r requirements-local.txt || true
USER superset

View File

@ -20,8 +20,7 @@ ARG VERSION
RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/incubator-superset.git /superset
WORKDIR /superset
# install doc dependencies
RUN pip install -r requirements.txt \
&& pip install -r docs/requirements.txt
RUN pip install -r requirements/documentation.txt
# build the docs
RUN python setup.py build_sphinx

View File

@ -86,8 +86,8 @@ SQLLAB_CTAS_NO_LIMIT = True
# the PYTHONPATH) in order to allow for local settings to be overridden
#
try:
from superset_config_docker import * # noqa
import superset_config_docker
from superset_config_docker import * # noqa
logger.info(
f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"

View File

@ -29,6 +29,7 @@
import os
import shlex
import sys
from typing import Any, Dict
import sphinx_rtd_theme
@ -130,7 +131,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {}
html_theme_options: Dict[str, Any] = {}
# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []
@ -221,7 +222,7 @@ htmlhelp_basename = "supersetdoc"
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
latex_elements: Dict[str, Any] = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').

View File

@ -1,104 +0,0 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file=requirements.txt setup.py
#
aiohttp==3.6.2 # via slackclient
alembic==1.4.2 # via flask-migrate
amqp==2.6.0 # via kombu
apispec[yaml]==3.3.1 # via flask-appbuilder
async-timeout==3.0.1 # via aiohttp
attrs==19.3.0 # via aiohttp, jsonschema
babel==2.8.0 # via flask-babel
backoff==1.10.0 # via apache-superset (setup.py)
billiard==3.6.3.0 # via celery
bleach==3.1.5 # via apache-superset (setup.py)
brotli==1.0.7 # via flask-compress
cachelib==0.1.1 # via apache-superset (setup.py)
celery==4.4.6 # via apache-superset (setup.py)
cffi==1.14.0 # via cryptography
chardet==3.0.4 # via aiohttp
click==7.1.2 # via apache-superset (setup.py), flask, flask-appbuilder
colorama==0.4.3 # via apache-superset (setup.py), flask-appbuilder
contextlib2==0.6.0.post1 # via apache-superset (setup.py)
croniter==0.3.31 # via apache-superset (setup.py)
cryptography==2.9.2 # via apache-superset (setup.py)
dataclasses==0.6 # via apache-superset (setup.py)
decorator==4.4.2 # via retry
defusedxml==0.6.0 # via python3-openid
dnspython==1.16.0 # via email-validator
email-validator==1.1.0 # via flask-appbuilder
flask-appbuilder==3.0.1 # via apache-superset (setup.py)
flask-babel==1.0.0 # via flask-appbuilder
flask-caching==1.8.0 # via apache-superset (setup.py)
flask-compress==1.5.0 # via apache-superset (setup.py)
flask-jwt-extended==3.24.1 # via flask-appbuilder
flask-login==0.4.1 # via flask-appbuilder
flask-migrate==2.5.3 # via apache-superset (setup.py)
flask-openid==1.2.5 # via flask-appbuilder
flask-sqlalchemy==2.4.1 # via flask-appbuilder, flask-migrate
flask-talisman==0.7.0 # via apache-superset (setup.py)
flask-wtf==0.14.3 # via apache-superset (setup.py), flask-appbuilder
flask==1.1.2 # via apache-superset (setup.py), flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf
future==0.18.2 # via celery
geographiclib==1.50 # via geopy
geopy==1.22.0 # via apache-superset (setup.py)
gunicorn==20.0.4 # via apache-superset (setup.py)
humanize==2.4.0 # via apache-superset (setup.py)
idna-ssl==1.1.0 # via aiohttp
idna==2.9 # via email-validator, idna-ssl, yarl
importlib-metadata==1.6.0 # via jsonschema, kombu, markdown
isodate==0.6.0 # via apache-superset (setup.py)
itsdangerous==1.1.0 # via flask, flask-wtf
jinja2==2.11.2 # via flask, flask-babel
jsonschema==3.2.0 # via flask-appbuilder
kombu==4.6.11 # via celery
mako==1.1.2 # via alembic
markdown==3.2.2 # via apache-superset (setup.py)
markupsafe==1.1.1 # via jinja2, mako, wtforms
marshmallow-enum==1.5.1 # via flask-appbuilder
marshmallow-sqlalchemy==0.23.0 # via flask-appbuilder
marshmallow==3.6.1 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
msgpack==1.0.0 # via apache-superset (setup.py)
multidict==4.7.6 # via aiohttp, yarl
numpy==1.18.4 # via pandas, pyarrow
packaging==20.3 # via bleach
pandas==1.0.3 # via apache-superset (setup.py)
parsedatetime==2.5 # via apache-superset (setup.py)
pathlib2==2.3.5 # via apache-superset (setup.py)
polyline==1.4.0 # via apache-superset (setup.py)
prison==0.1.3 # via flask-appbuilder
py==1.8.1 # via retry
pyarrow==0.17.0 # via apache-superset (setup.py)
pycparser==2.20 # via cffi
pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended
pyparsing==2.4.7 # via packaging
pyrsistent==0.16.0 # via jsonschema
python-dateutil==2.8.1 # via alembic, apache-superset (setup.py), croniter, flask-appbuilder, pandas
python-dotenv==0.13.0 # via apache-superset (setup.py)
python-editor==1.0.4 # via alembic
python-geohash==0.8.5 # via apache-superset (setup.py)
python3-openid==3.1.0 # via flask-openid
pytz==2020.1 # via babel, celery, flask-babel, pandas
pyyaml==5.3.1 # via apache-superset (setup.py), apispec
retry==0.9.2 # via apache-superset (setup.py)
selenium==3.141.0 # via apache-superset (setup.py)
simplejson==3.17.0 # via apache-superset (setup.py)
six==1.14.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
slackclient==2.5.0 # via apache-superset (setup.py)
sqlalchemy-utils==0.36.6 # via apache-superset (setup.py), flask-appbuilder
sqlalchemy==1.3.16 # via alembic, apache-superset (setup.py), flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlparse==0.3.0 # via apache-superset (setup.py)
typing-extensions==3.7.4.2 # via aiohttp
urllib3==1.25.9 # via selenium
vine==1.3.0 # via amqp, celery
webencodings==0.5.1 # via bleach
werkzeug==1.0.1 # via flask, flask-jwt-extended
wtforms-json==0.3.3 # via apache-superset (setup.py)
wtforms==2.3.1 # via flask-wtf, wtforms-json
yarl==1.4.2 # via aiohttp
zipp==3.1.0 # via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:
# setuptools

7
superset/bin/superset → requirements/base.in Executable file → Normal file
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@ -15,7 +15,4 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from superset.cli import superset
if __name__ == "__main__":
superset()
-e file:.[cors,druid,hive,mysql,postgres,thumbnails]

135
requirements/base.txt Normal file
View File

@ -0,0 +1,135 @@
# SHA1:3b8e2a2d526726ac7ff836060bf89101ff52c1c8
#
# This file is autogenerated by pip-compile-multi
# To update, run:
#
# pip-compile-multi
#
-e file:. # via -r requirements/base.in
aiohttp==3.6.2 # via slackclient
alembic==1.4.2 # via flask-migrate
amqp==2.6.1 # via kombu
apispec[yaml]==3.3.1 # via flask-appbuilder
async-timeout==3.0.1 # via aiohttp
attrs==19.3.0 # via aiohttp, jsonschema
babel==2.8.0 # via flask-babel
backoff==1.10.0 # via apache-superset
billiard==3.6.3.0 # via celery
bleach==3.1.5 # via apache-superset
boto3==1.14.34 # via tabulator
botocore==1.17.34 # via boto3, s3transfer
brotli==1.0.7 # via flask-compress
cached-property==1.5.1 # via tableschema
cachelib==0.1.1 # via apache-superset
celery==4.4.7 # via apache-superset
certifi==2020.6.20 # via requests
cffi==1.14.1 # via cryptography
chardet==3.0.4 # via aiohttp, requests, tabulator
click==7.1.2 # via apache-superset, flask, flask-appbuilder, tableschema, tabulator
colorama==0.4.3 # via apache-superset, flask-appbuilder
contextlib2==0.6.0.post1 # via apache-superset
croniter==0.3.34 # via apache-superset
cryptography==3.0 # via apache-superset
dataclasses==0.6 # via apache-superset
decorator==4.4.2 # via retry
defusedxml==0.6.0 # via python3-openid
dnspython==2.0.0 # via email-validator
docutils==0.15.2 # via botocore
email-validator==1.1.1 # via flask-appbuilder
et-xmlfile==1.0.1 # via openpyxl
flask-appbuilder==3.0.1 # via apache-superset
flask-babel==1.0.0 # via flask-appbuilder
flask-caching==1.9.0 # via apache-superset
flask-compress==1.5.0 # via apache-superset
flask-cors==3.0.8 # via apache-superset
flask-jwt-extended==3.24.1 # via flask-appbuilder
flask-login==0.4.1 # via flask-appbuilder
flask-migrate==2.5.3 # via apache-superset
flask-openid==1.2.5 # via flask-appbuilder
flask-sqlalchemy==2.4.4 # via flask-appbuilder, flask-migrate
flask-talisman==0.7.0 # via apache-superset
flask-wtf==0.14.3 # via apache-superset, flask-appbuilder
flask==1.1.2 # via apache-superset, flask-appbuilder, flask-babel, flask-caching, flask-compress, flask-cors, flask-jwt-extended, flask-login, flask-migrate, flask-openid, flask-sqlalchemy, flask-wtf
future==0.18.2 # via pyhive
geographiclib==1.50 # via geopy
geopy==2.0.0 # via apache-superset
gunicorn==20.0.4 # via apache-superset
humanize==2.5.0 # via apache-superset
idna-ssl==1.1.0 # via aiohttp
idna==2.10 # via email-validator, idna-ssl, requests, yarl
ijson==3.1.post0 # via tabulator
importlib-metadata==1.7.0 # via jsonschema, kombu, markdown
isodate==0.6.0 # via apache-superset, tableschema
itsdangerous==1.1.0 # via flask, flask-wtf
jdcal==1.4.1 # via openpyxl
jinja2==2.11.2 # via flask, flask-babel
jmespath==0.10.0 # via boto3, botocore
jsonlines==1.2.0 # via tabulator
jsonschema==3.2.0 # via flask-appbuilder, tableschema
kombu==4.6.11 # via celery
linear-tsv==1.1.0 # via tabulator
mako==1.1.3 # via alembic
markdown==3.2.2 # via apache-superset
markupsafe==1.1.1 # via jinja2, mako, wtforms
marshmallow-enum==1.5.1 # via flask-appbuilder
marshmallow-sqlalchemy==0.23.1 # via flask-appbuilder
marshmallow==3.7.1 # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
msgpack==1.0.0 # via apache-superset
multidict==4.7.6 # via aiohttp, yarl
mysqlclient==1.4.2.post1 # via apache-superset
natsort==7.0.1 # via croniter
numpy==1.19.1 # via pandas, pyarrow
openpyxl==3.0.4 # via tabulator
packaging==20.4 # via bleach
pandas==1.0.5 # via apache-superset
parsedatetime==2.6 # via apache-superset
pathlib2==2.3.5 # via apache-superset
pillow==7.2.0 # via apache-superset
polyline==1.4.0 # via apache-superset
prison==0.1.3 # via flask-appbuilder
psycopg2-binary==2.8.5 # via apache-superset
py==1.9.0 # via retry
pyarrow==0.17.1 # via apache-superset
pycparser==2.20 # via cffi
pydruid==0.6.1 # via apache-superset
pyhive[hive]==0.6.2 # via apache-superset
pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended
pyparsing==2.4.7 # via packaging
pyrsistent==0.16.0 # via jsonschema
python-dateutil==2.8.1 # via alembic, apache-superset, botocore, croniter, flask-appbuilder, pandas, pyhive, tableschema
python-dotenv==0.14.0 # via apache-superset
python-editor==1.0.4 # via alembic
python-geohash==0.8.5 # via apache-superset
python3-openid==3.2.0 # via flask-openid
pytz==2020.1 # via babel, celery, flask-babel, pandas
pyyaml==5.3.1 # via apache-superset, apispec
requests==2.24.0 # via pydruid, tableschema, tabulator
retry==0.9.2 # via apache-superset
rfc3986==1.4.0 # via tableschema
s3transfer==0.3.3 # via boto3
sasl==0.2.1 # via pyhive, thrift-sasl
selenium==3.141.0 # via apache-superset
simplejson==3.17.2 # via apache-superset
six==1.15.0 # via bleach, cryptography, flask-cors, flask-jwt-extended, flask-talisman, isodate, jsonlines, jsonschema, linear-tsv, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sasl, sqlalchemy-utils, tableschema, tabulator, thrift, thrift-sasl, wtforms-json
slackclient==2.5.0 # via apache-superset
sqlalchemy-utils==0.36.8 # via apache-superset, flask-appbuilder
sqlalchemy==1.3.18 # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils, tabulator
sqlparse==0.3.0 # via apache-superset
tableschema==1.19.2 # via apache-superset
tabulator==1.52.3 # via tableschema
thrift-sasl==0.4.2 # via pyhive
thrift==0.13.0 # via apache-superset, pyhive, thrift-sasl
typing-extensions==3.7.4.2 # via aiohttp, yarl
unicodecsv==0.14.1 # via tableschema, tabulator
urllib3==1.25.10 # via botocore, requests, selenium
vine==1.3.0 # via amqp, celery
webencodings==0.5.1 # via bleach
werkzeug==1.0.1 # via flask, flask-jwt-extended
wtforms-json==0.3.3 # via apache-superset
wtforms==2.3.3 # via flask-wtf, wtforms-json
xlrd==1.2.0 # via tabulator
yarl==1.5.1 # via aiohttp
zipp==3.1.0 # via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:
# setuptools

View File

@ -14,5 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
sphinx==3.0.1
sphinx-rtd-theme==0.4.3
-r base.in
sphinx
sphinx-rtd-theme

View File

@ -0,0 +1,24 @@
# SHA1:2cf3a7081bf9189afa931880771c21608989698b
#
# This file is autogenerated by pip-compile-multi
# To update, run:
#
# pip-compile-multi
#
-r base.txt
-e file:. # via -r requirements/base.in
alabaster==0.7.12 # via sphinx
imagesize==1.2.0 # via sphinx
pygments==2.6.1 # via sphinx
snowballstemmer==2.0.0 # via sphinx
sphinx-rtd-theme==0.5.0 # via -r requirements/documentation.in
sphinx==3.1.2 # via -r requirements/documentation.in, sphinx-rtd-theme
sphinxcontrib-applehelp==1.0.2 # via sphinx
sphinxcontrib-devhelp==1.0.2 # via sphinx
sphinxcontrib-htmlhelp==1.0.3 # via sphinx
sphinxcontrib-jsmath==1.0.1 # via sphinx
sphinxcontrib-qthelp==1.0.3 # via sphinx
sphinxcontrib-serializinghtml==1.1.4 # via sphinx
# The following packages are considered to be unsafe in a requirements file:
# setuptools

View File

@ -14,28 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
black==19.10b0
coverage==4.5.3
flask-cors==3.0.7
flask-testing==0.8.0
ipdb==0.12
isort==4.3.21
mypy==0.770
openapi-spec-validator==0.2.8
pytest==5.4.3
pytest-cov==2.10.0
parameterized==0.7.4
pip-tools==5.1.2
pre-commit==1.17.0
psycopg2-binary==2.8.5
pycodestyle==2.5.0
pydruid==0.6.1
pyhive==0.6.2
pylint==2.5.3
redis==3.5.1
requests==2.23.0
statsd==3.3.0
tox==3.11.1
pillow==7.0.0
openpyxl==3.0.3 # Pandas use openpyxl to write excel format(using in unittes)
xlrd==1.2.0
pip-compile-multi!=1.5.9
pre-commit
tox

View File

@ -0,0 +1,33 @@
# SHA1:2e1bdbb387f30da2c99c8ca9cbf7ce42efba84cf
#
# This file is autogenerated by pip-compile-multi
# To update, run:
#
# pip-compile-multi
#
appdirs==1.4.4 # via virtualenv
cfgv==3.2.0 # via pre-commit
click==7.1.2 # via pip-compile-multi, pip-tools
distlib==0.3.1 # via virtualenv
filelock==3.0.12 # via tox, virtualenv
identify==1.4.25 # via pre-commit
importlib-metadata==1.7.0 # via pluggy, pre-commit, tox, virtualenv
importlib-resources==3.0.0 # via pre-commit, virtualenv
nodeenv==1.4.0 # via pre-commit
packaging==20.4 # via tox
pip-compile-multi==1.5.8 # via -r requirements/integration.in
pip-tools==5.3.1 # via pip-compile-multi
pluggy==0.13.1 # via tox
pre-commit==2.6.0 # via -r requirements/integration.in
py==1.9.0 # via tox
pyparsing==2.4.7 # via packaging
pyyaml==5.3.1 # via pre-commit
six==1.15.0 # via packaging, pip-tools, tox, virtualenv
toml==0.10.1 # via pre-commit, tox
toposort==1.5 # via pip-compile-multi
tox==3.18.1 # via -r requirements/integration.in
virtualenv==20.0.29 # via pre-commit, tox
zipp==3.1.0 # via importlib-metadata, importlib-resources
# The following packages are considered to be unsafe in a requirements file:
# pip

17
requirements/local.in Normal file
View File

@ -0,0 +1,17 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
-r base.in

12
requirements/local.txt Normal file
View File

@ -0,0 +1,12 @@
# SHA1:37fdd6cd1e7782cad40e1c59a0b7b6e1d9f73332
#
# This file is autogenerated by pip-compile-multi
# To update, run:
#
# pip-compile-multi
#
-r base.txt
-e file:. # via -r requirements/base.in
# The following packages are considered to be unsafe in a requirements file:
# setuptools

27
requirements/testing.in Normal file
View File

@ -0,0 +1,27 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
-r base.in
-r integration.in
flask-testing
openapi-spec-validator
openpyxl
parameterized
pylint
pytest
pytest-cov
redis
statsd

31
requirements/testing.txt Normal file
View File

@ -0,0 +1,31 @@
# SHA1:785ae7ffcde3cee8ebcc0a839cdb8e61e693d329
#
# This file is autogenerated by pip-compile-multi
# To update, run:
#
# pip-compile-multi
#
-r base.txt
-r integration.txt
-e file:. # via -r requirements/base.in
astroid==2.4.2 # via pylint
coverage==5.2.1 # via pytest-cov
flask-testing==0.8.0 # via -r requirements/testing.in
iniconfig==1.0.1 # via pytest
isort==4.3.21 # via pylint
lazy-object-proxy==1.4.3 # via astroid
mccabe==0.6.1 # via pylint
more-itertools==8.4.0 # via pytest
openapi-spec-validator==0.2.9 # via -r requirements/testing.in
parameterized==0.7.4 # via -r requirements/testing.in
pylint==2.5.3 # via -r requirements/testing.in
pytest-cov==2.10.0 # via -r requirements/testing.in
pytest==6.0.1 # via -r requirements/testing.in, pytest-cov
redis==3.5.3 # via -r requirements/testing.in
statsd==3.3.0 # via -r requirements/testing.in
typed-ast==1.4.1 # via astroid
wrapt==1.12.1 # via astroid
# The following packages are considered to be unsafe in a requirements file:
# pip
# setuptools

View File

@ -16,12 +16,14 @@
# under the License.
from collections import defaultdict
from superset import sm
from superset import security_manager
def cleanup_permissions():
# 1. Clean up duplicates.
pvms = sm.get_session.query(sm.permissionview_model).all()
pvms = security_manager.get_session.query(
security_manager.permissionview_model
).all()
print("# of permission view menues is: {}".format(len(pvms)))
pvms_dict = defaultdict(list)
for pvm in pvms:
@ -34,34 +36,42 @@ def cleanup_permissions():
roles = set(first_prm.role)
for pvm in pvm_list[1:]:
roles = roles.union(pvm.role)
sm.get_session.delete(pvm)
security_manager.get_session.delete(pvm)
first_prm.roles = list(roles)
sm.get_session.commit()
security_manager.get_session.commit()
pvms = sm.get_session.query(sm.permissionview_model).all()
print("STage 1: # of permission view menues is: {}".format(len(pvms)))
pvms = security_manager.get_session.query(
security_manager.permissionview_model
).all()
print("Stage 1: # of permission view menues is: {}".format(len(pvms)))
# 2. Clean up None permissions or view menues
pvms = sm.get_session.query(sm.permissionview_model).all()
pvms = security_manager.get_session.query(
security_manager.permissionview_model
).all()
for pvm in pvms:
if not (pvm.view_menu and pvm.permission):
sm.get_session.delete(pvm)
sm.get_session.commit()
security_manager.get_session.delete(pvm)
security_manager.get_session.commit()
pvms = sm.get_session.query(sm.permissionview_model).all()
pvms = security_manager.get_session.query(
security_manager.permissionview_model
).all()
print("Stage 2: # of permission view menues is: {}".format(len(pvms)))
# 3. Delete empty permission view menues from roles
roles = sm.get_session.query(sm.role_model).all()
roles = security_manager.get_session.query(security_manager.role_model).all()
for role in roles:
role.permissions = [p for p in role.permissions if p]
sm.get_session.commit()
security_manager.get_session.commit()
# 4. Delete empty roles from permission view menues
pvms = sm.get_session.query(sm.permissionview_model).all()
pvms = security_manager.get_session.query(
security_manager.permissionview_model
).all()
for pvm in pvms:
pvm.role = [r for r in pvm.role if r]
sm.get_session.commit()
security_manager.get_session.commit()
cleanup_permissions()

View File

@ -52,3 +52,6 @@ warn_unused_ignores = true
check_untyped_defs = true
disallow_untyped_calls = true
disallow_untyped_defs = true
[mypy-superset.migrations.versions.*]
ignore_errors = true

View File

@ -65,7 +65,7 @@ setup(
packages=find_packages(),
include_package_data=True,
zip_safe=False,
scripts=["superset/bin/superset"],
entry_points={"console_scripts": ["superset=superset.cli:superset"]},
install_requires=[
"backoff>=1.8.0",
"bleach>=3.0.2, <4.0.0",
@ -135,5 +135,4 @@ setup(
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
],
tests_require=["flask-testing==0.8.0"],
)

View File

@ -124,63 +124,62 @@ class SupersetAppInitializer:
#
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
from superset.charts.api import ChartRestApi
from superset.connectors.druid.views import (
DruidDatasourceModelView,
DruidClusterModelView,
DruidMetricInlineView,
DruidColumnInlineView,
Druid,
DruidClusterModelView,
DruidColumnInlineView,
DruidDatasourceModelView,
DruidMetricInlineView,
)
from superset.connectors.sqla.views import (
RowLevelSecurityFiltersModelView,
SqlMetricInlineView,
TableColumnInlineView,
TableModelView,
)
from superset.dashboards.api import DashboardRestApi
from superset.databases.api import DatabaseRestApi
from superset.datasets.api import DatasetRestApi
from superset.queries.api import QueryRestApi
from superset.connectors.sqla.views import (
TableColumnInlineView,
SqlMetricInlineView,
TableModelView,
RowLevelSecurityFiltersModelView,
)
from superset.views.access_requests import AccessRequestsModelView
from superset.views.alerts import AlertLogModelView, AlertModelView
from superset.views.annotations import (
AnnotationLayerModelView,
AnnotationModelView,
)
from superset.views.api import Api
from superset.views.chart.views import SliceAsync, SliceModelView
from superset.views.core import Superset
from superset.views.redirects import R
from superset.views.key_value import KV
from superset.views.access_requests import AccessRequestsModelView
from superset.views.css_templates import CssTemplateAsyncModelView
from superset.views.css_templates import CssTemplateModelView
from superset.charts.api import ChartRestApi
from superset.views.chart.views import SliceModelView, SliceAsync
from superset.dashboards.api import DashboardRestApi
from superset.views.css_templates import (
CssTemplateAsyncModelView,
CssTemplateModelView,
)
from superset.views.dashboard.views import (
DashboardModelView,
Dashboard,
DashboardModelView,
DashboardModelViewAsync,
)
from superset.databases.api import DatabaseRestApi
from superset.views.database.views import (
DatabaseView,
CsvToDatabaseView,
DatabaseView,
ExcelToDatabaseView,
)
from superset.views.datasource import Datasource
from superset.views.key_value import KV
from superset.views.log.api import LogRestApi
from superset.views.log.views import LogModelView
from superset.views.redirects import R
from superset.views.schedules import (
DashboardEmailScheduleView,
SliceEmailScheduleView,
)
from superset.views.alerts import (
AlertModelView,
AlertLogModelView,
)
from superset.views.sql_lab import (
SavedQueryViewApi,
SavedQueryView,
TabStateView,
TableSchemaView,
SavedQueryViewApi,
SqlLab,
TableSchemaView,
TabStateView,
)
from superset.views.tags import TagView
@ -541,8 +540,10 @@ class SupersetAppInitializer:
# Doing local imports here as model importing causes a reference to
# app.config to be invoked and we need the current_app to have been setup
#
from superset.utils.url_map_converters import RegexConverter
from superset.utils.url_map_converters import ObjectTypeConverter
from superset.utils.url_map_converters import (
ObjectTypeConverter,
RegexConverter,
)
self.flask_app.url_map.converters["regex"] = RegexConverter
self.flask_app.url_map.converters["object_type"] = ObjectTypeConverter

View File

@ -402,7 +402,7 @@ class ChartDataSelectOptionsSchema(ChartDataPostProcessingOperationOptionsSchema
"referenced here.",
example=["country", "gender", "age"],
)
exclude = fields.List( # type: ignore
exclude = fields.List(
fields.String(),
description="Columns to exclude from selection.",
example=["my_temp_column"],

View File

@ -613,8 +613,8 @@ def sync_tags() -> None:
def alert() -> None:
"""Run the alert scheduler loop"""
# this command is just for testing purposes
from superset.tasks.schedules import schedule_window
from superset.models.schedules import ScheduleType
from superset.tasks.schedules import schedule_window
click.secho("Processing one alert loop", fg="green")
schedule_window(

View File

@ -49,7 +49,7 @@ class CommandInvalidError(CommandException):
def normalized_messages(self) -> Dict[Any, Any]:
errors: Dict[Any, Any] = {}
for exception in self._invalid_exceptions:
errors.update(exception.normalized_messages()) # type: ignore
errors.update(exception.normalized_messages())
return errors

View File

@ -49,6 +49,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from flask_appbuilder.security.sqla import models # pylint: disable=unused-import
from superset.models.core import Database # pylint: disable=unused-import
# Realtime stats logger, a StatsD implementation exists
@ -918,8 +919,8 @@ if CONFIG_PATH_ENV_VAR in os.environ:
raise
elif importlib.util.find_spec("superset_config"):
try:
from superset_config import * # pylint: disable=import-error,wildcard-import,unused-wildcard-import
import superset_config # pylint: disable=import-error
from superset_config import * # type: ignore # pylint: disable=import-error,wildcard-import,unused-wildcard-import
print(f"Loaded your LOCAL configuration at [{superset_config.__file__}]")
except Exception:

View File

@ -25,8 +25,8 @@ if TYPE_CHECKING:
# pylint: disable=unused-import
from collections import OrderedDict
from superset.models.core import Database
from superset.connectors.base.models import BaseDatasource
from superset.models.core import Database
class ConnectorRegistry:

View File

@ -59,6 +59,7 @@ from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict
from superset.utils import core as utils, import_datasource
try:
import requests
from pydruid.client import PyDruid
from pydruid.utils.aggregators import count
from pydruid.utils.dimensions import (
@ -77,7 +78,6 @@ try:
Quantile,
Quantiles,
)
import requests
except ImportError:
pass

View File

@ -89,13 +89,14 @@ class HiveEngineSpec(PrestoEngineSpec):
@classmethod
def patch(cls) -> None:
from pyhive import hive # pylint: disable=no-name-in-module
from superset.db_engines import hive as patched_hive
from TCLIService import (
constants as patched_constants,
ttypes as patched_ttypes,
TCLIService as patched_TCLIService,
ttypes as patched_ttypes,
)
from superset.db_engines import hive as patched_hive
hive.TCLIService = patched_TCLIService
hive.constants = patched_constants
hive.ttypes = patched_ttypes

View File

@ -1374,8 +1374,8 @@ class MultiLineViz(NVD3Viz):
def get_data(self, df: pd.DataFrame) -> VizData:
fd = self.form_data
# Late imports to avoid circular import issues
from superset.models.slice import Slice
from superset import db
from superset.models.slice import Slice
slice_ids1 = fd.get("line_charts")
slices1 = db.session.query(Slice).filter(Slice.id.in_(slice_ids1)).all()
@ -2284,8 +2284,8 @@ class DeckGLMultiLayer(BaseViz):
def get_data(self, df: pd.DataFrame) -> VizData:
fd = self.form_data
# Late imports to avoid circular import issues
from superset.models.slice import Slice
from superset import db
from superset.models.slice import Slice
slice_ids = fd.get("deck_slices")
slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all()

View File

@ -1356,8 +1356,8 @@ class MultiLineViz(NVD3Viz):
def get_data(self, df: pd.DataFrame) -> VizData:
fd = self.form_data
# Late imports to avoid circular import issues
from superset.models.slice import Slice
from superset import db
from superset.models.slice import Slice
slice_ids1 = fd.get("line_charts")
slices1 = db.session.query(Slice).filter(Slice.id.in_(slice_ids1)).all()
@ -2174,8 +2174,8 @@ class DeckGLMultiLayer(BaseViz):
def get_data(self, df: pd.DataFrame) -> VizData:
fd = self.form_data
# Late imports to avoid circular import issues
from superset.models.slice import Slice
from superset import db
from superset.models.slice import Slice
slice_ids = fd.get("deck_slices")
slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all()

51
tox.ini
View File

@ -23,10 +23,7 @@ commands =
# no args or tests/* can be passed as an argument to run all tests
pytest --ignore=load_examples_test {posargs}
deps =
-rrequirements.txt
-rrequirements-dev.txt
mysql: .[mysql]
postgres: .[postgres]
-rrequirements/testing.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_CONFIG = tests.superset_test_config
@ -37,20 +34,13 @@ setenv =
whitelist_externals =
npm
[testenv:black]
commands =
black --check setup.py superset tests
deps =
-rrequirements-dev.txt
[testenv:cypress-dashboard]
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh dashboard
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rrequirements/testing.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_CONFIG = tests.superset_test_config
@ -62,8 +52,7 @@ commands =
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh explore
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rrequirements/testing.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_CONFIG = tests.superset_test_config
@ -75,8 +64,7 @@ commands =
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh sqllab
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rrequirements/testing.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_CONFIG = tests.superset_test_config
@ -88,8 +76,7 @@ commands =
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh sqllab
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rrequirements/testing.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_CONFIG = tests.superset_test_config_sqllab_backend_persist
@ -99,9 +86,7 @@ setenv =
commands =
sphinx-build -b html docs _build/html -W
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rdocs/requirements.txt
-rrequirements/documentation.txt
[testenv:eslint]
changedir = {toxinidir}/superset-frontend
@ -115,12 +100,6 @@ commands =
deps =
passenv = *
[testenv:isort]
commands =
isort --check-only --recursive setup.py superset tests
deps =
-rrequirements-dev.txt
[testenv:javascript]
commands =
npm install -g npm@'>=6.5.0'
@ -135,30 +114,27 @@ whitelist_externals =
{toxinidir}/scripts/check_license.sh
deps =
[testenv:mypy]
[testenv:pre-commit]
commands =
mypy setup.py superset tests
pre-commit run --all-files
deps =
-rrequirements-dev.txt
-rrequirements/integration.txt
skip_install = true
[testenv:pylint]
commands =
pylint superset
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rrequirements/testing.txt
[testenv:thumbnails]
setenv =
SUPERSET_CONFIG = tests.superset_test_config_thumbnails
deps =
-rrequirements.txt
-rrequirements-dev.txt
.[postgres]
-rrequirements/testing.txt
[tox]
envlist =
black
cypress-dashboard
cypress-explore
cypress-sqllab
@ -166,9 +142,8 @@ envlist =
docs
eslint
fossa
isort
javascript
license-check
mypy
pre-commit
pylint
skipsdist = true