chore: consolidate the Superset python package metadata (#27884)

This commit is contained in:
Maxime Beauchemin 2024-04-15 14:44:47 -07:00 committed by GitHub
parent 99c414e4da
commit c225e17a75
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 216 additions and 246 deletions

View File

@ -76,7 +76,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/*
COPY --chown=superset:superset setup.py MANIFEST.in README.md ./
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
# setup.py uses the version information in package.json
COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
COPY --chown=superset:superset requirements/base.txt requirements/

View File

@ -70,7 +70,10 @@ If you have made changes to the FAB-managed templates, which are not built the s
#### Dependencies
If you add a new requirement or update an existing requirement (per the `install_requires` section in `setup.py`) you must recompile (freeze) the Python dependencies to ensure that for CI, testing, etc. the build is deterministic. This can be achieved via,
If you add a new requirement or update an existing requirement (per the
`requirements` section in `pyproject.toml`) you must recompile (freeze) the
Python dependencies to ensure that for CI, testing, etc. the build is deterministic.
This can be achieved via,
```bash
$ python3 -m venv venv

View File

@ -54,7 +54,9 @@ Finally, never submit a PR that will put master branch in broken state. If the P
- Recommended capture tools ([Kap](https://getkap.co/), [LICEcap](https://www.cockos.com/licecap/), [Skitch](https://download.cnet.com/Skitch/3000-13455_4-189876.html))
- If no screenshot is provided, the committers will mark the PR with `need:screenshot` label and will not review until screenshot is provided.
- **Dependencies:** Be careful about adding new dependency and avoid unnecessary dependencies.
- For Python, include it in `setup.py` denoting any specific restrictions and in `requirements.txt` pinned to a specific version which ensures that the application build is deterministic.
- For Python, include it in `pyproject.toml` denoting any specific restrictions and
in `requirements.txt` pinned to a specific version which ensures that the application
build is deterministic.
- For TypeScript/JavaScript, include new libraries in `package.json`
- **Tests:** The pull request should include tests, either as doctests, unit tests, or both. Make sure to resolve all errors and test failures. See [Testing](#testing) for how to run tests.
- **Documentation:** If the pull request adds functionality, the docs should be updated as part of the same PR.

View File

@ -20,7 +20,9 @@ which is part of the Python standard library.
Youll need to install the required packages for the database you want to use as your metadata database
as well as the packages needed to connect to the databases you want to access through Superset.
Some of the recommended packages are shown below. Please refer to [setup.py](https://github.com/apache/superset/blob/master/setup.py) for the versions that are compatible with Superset.
Some of the recommended packages are shown below. Please refer to
[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) for the versions that
are compatible with Superset.
| Database | PyPI package | Connection String |
| --------------------------------------------------------- | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |

View File

@ -66,7 +66,11 @@ We don't recommend using the system installed Python. Instead, first install the
brew install readline pkg-config libffi openssl mysql postgresql@14
```
You should install a recent version of Python. Refer to the [setup.py](https://github.com/apache/superset/blob/master/setup.py) file for a list of Python versions officially supported by Superset. We'd recommend using a Python version manager like [pyenv](https://github.com/pyenv/pyenv) (and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
You should install a recent version of Python. Refer to the
[pyproject.toml](https://github.com/apache/superset/blob/master/pyproject.toml) file for a list of Python
versions officially supported by Superset. We'd recommend using a Python version manager
like [pyenv](https://github.com/pyenv/pyenv)
(and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
:::tip
To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py).

View File

@ -23,7 +23,9 @@ build-backend = "setuptools.build_meta"
name = "apache-superset"
description = "A modern, enterprise-ready business intelligence web application"
readme = "README.md"
dynamic = ["version", "scripts", "entry-points", "license", "requires-python"]
dynamic = ["version", "scripts", "entry-points"]
requires-python = "~=3.9"
license = { file="LICENSE.txt" }
authors = [
{ name = "Apache Software Foundation", email = "dev@superset.apache.org" },
]
@ -197,3 +199,196 @@ development = [
[project.urls]
homepage = "https://superset.apache.org/"
documentation = "https://superset.apache.org/docs/intro"
[tool.isort]
combine_as_imports = true
include_trailing_comma = true
line_length = 88
known_first_party = "superset"
known_third_party = "alembic, apispec, backoff, celery, click, colorama, cron_descriptor, croniter, cryptography, dateutil, deprecation, flask, flask_appbuilder, flask_babel, flask_caching, flask_compress, flask_jwt_extended, flask_login, flask_migrate, flask_sqlalchemy, flask_talisman, flask_testing, flask_wtf, freezegun, geohash, geopy, holidays, humanize, isodate, jinja2, jwt, markdown, markupsafe, marshmallow, msgpack, nh3, numpy, pandas, parameterized, parsedatetime, pgsanity, pkg_resources, polyline, prison, progress, pyarrow, sqlalchemy_bigquery, pyhive, pyparsing, pytest, pytest_mock, pytz, redis, requests, selenium, setuptools, shillelagh, simplejson, slack, sqlalchemy, sqlalchemy_utils, sqlparse, typing_extensions, urllib3, werkzeug, wtforms, wtforms_json, yaml"
multi_line_output = 3
order_by_type = false
[tool.mypy]
check_untyped_defs = true
disallow_any_generics = true
disallow_untyped_calls = true
disallow_untyped_defs = true
ignore_missing_imports = true
no_implicit_optional = true
warn_unused_ignores = true
[[tool.mypy.overrides]]
module = "superset.migrations.versions.*"
ignore_errors = true
[[tool.mypy.overrides]]
module = "tests.*"
check_untyped_defs = false
disallow_untyped_calls = false
disallow_untyped_defs = false
[tool.tox]
legacy_tox_ini = """
# Remember to start celery workers to run celery tests, e.g.
# celery --app=superset.tasks.celery_app:app worker -Ofair -c 2
[testenv]
basepython = python3.10
ignore_basepython_conflict = true
commands =
superset db upgrade
superset init
# use -s to be able to use break pointers.
# no args or tests/* can be passed as an argument to run all tests
pytest -s {posargs}
deps =
-rrequirements/development.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test
sqlite:
SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db
SUPERSET__SQLALCHEMY_EXAMPLES_URI = sqlite:////{envtmpdir}/examples.db
mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
# docker run -p 8080:8080 --name presto starburstdata/presto
mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default
# based on https://github.com/big-data-europe/docker-hadoop
# clone the repo & run docker compose up -d to test locally
mysql-hive: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
mysql-hive: SUPERSET__SQLALCHEMY_EXAMPLES_URI = hive://localhost:10000/default
# make sure that directory is accessible by docker
hive: UPLOAD_FOLDER = /tmp/.superset/app/static/uploads/
usedevelop = true
allowlist_externals =
npm
pkill
[testenv:cypress]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-dashboard]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh dashboard
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-explore]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh explore
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-sqllab]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh sqllab
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-sqllab-backend-persist]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh sqllab
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:eslint]
changedir = {toxinidir}/superset-frontend
commands =
npm run lint
deps =
[testenv:fossa]
commands =
{toxinidir}/scripts/fossa.sh
deps =
passenv = *
[testenv:javascript]
commands =
npm install -g npm@'>=6.5.0'
{toxinidir}/superset-frontend/js_build.sh
deps =
[testenv:license-check]
commands =
{toxinidir}/scripts/check_license.sh
passenv = *
whitelist_externals =
{toxinidir}/scripts/check_license.sh
deps =
[testenv:pre-commit]
commands =
pre-commit run --all-files
deps =
-rrequirements/development.txt
skip_install = true
[testenv:pylint]
commands =
pylint superset
deps =
-rrequirements/development.txt
[testenv:thumbnails]
setenv =
SUPERSET_CONFIG = tests.integration_tests.superset_test_config_thumbnails
deps =
-rrequirements/development.txt
[tox]
envlist =
cypress-dashboard
cypress-explore
cypress-sqllab
cypress-sqllab-backend-persist
eslint
fossa
javascript
license-check
pre-commit
pylint
skipsdist = true
"""

View File

@ -1,52 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[metadata]
name = Superset
summary = a data exploration platform
description_file = README.md
author = Apache Superset Dev
author_email = dev@superset.apache.org
license = Apache License, Version 2.0
[files]
packages = superset
[isort]
combine_as_imports = true
include_trailing_comma = true
line_length = 88
known_first_party = superset
known_third_party =alembic,apispec,backoff,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,msgpack,nh3,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,sqlalchemy_bigquery,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,shillelagh,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml
multi_line_output = 3
order_by_type = false
[mypy]
check_untyped_defs = true
disallow_any_generics = true
disallow_untyped_calls = true
disallow_untyped_defs = true
ignore_missing_imports = true
no_implicit_optional = true
warn_unused_ignores = true
[mypy-superset.migrations.versions.*]
ignore_errors = true
[mypy-tests.*]
check_untyped_defs = false
disallow_untyped_calls = false
disallow_untyped_defs = false

View File

@ -23,12 +23,10 @@ from setuptools import find_packages, setup
BASE_DIR = os.path.abspath(os.path.dirname(__file__))
PACKAGE_JSON = os.path.join(BASE_DIR, "superset-frontend", "package.json")
with open(PACKAGE_JSON) as package_file:
version_string = json.load(package_file)["version"]
with open("README.md", encoding="utf-8") as f:
long_description = f.read()
def get_git_sha() -> str:
try:
@ -54,10 +52,6 @@ with open(VERSION_INFO_FILE, "w") as version_file:
version_string = version_string.replace("-dev", ".dev0")
setup(
name="apache-superset",
description="A modern, enterprise-ready business intelligence web application",
long_description=long_description,
long_description_content_type="text/markdown",
version=version_string,
packages=find_packages(),
include_package_data=True,
@ -75,6 +69,5 @@ setup(
"superset=superset.extensions.metadb:SupersetShillelaghAdapter"
],
},
python_requires="~=3.9",
download_url="https://www.apache.org/dist/superset/" + version_string,
)

View File

@ -30,7 +30,7 @@ def retry_call( # pylint: disable=too-many-arguments
giveup_log_level: int = logging.WARNING,
fargs: Optional[list[Any]] = None,
fkwargs: Optional[dict[str, Any]] = None,
**kwargs: Any
**kwargs: Any,
) -> Any:
"""
Retry a given call.

View File

@ -185,7 +185,7 @@ def create_dashboard_report(dashboard, extra, **kwargs):
extra={
"dashboard": extra,
},
**kwargs
**kwargs,
)
error = None

View File

@ -30,7 +30,7 @@ class CustomEncFieldAdapter(AbstractEncryptedFieldAdapter):
self,
app_config: Optional[dict[str, Any]],
*args: list[Any],
**kwargs: Optional[dict[str, Any]]
**kwargs: Optional[dict[str, Any]],
) -> TypeDecorator:
if app_config:
return StringEncryptedType(*args, app_config["SECRET_KEY"], **kwargs)

177
tox.ini
View File

@ -1,177 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Remember to start celery workers to run celery tests, e.g.
# celery --app=superset.tasks.celery_app:app worker -Ofair -c 2
[testenv]
basepython = python3.10
ignore_basepython_conflict = true
commands =
superset db upgrade
superset init
# use -s to be able to use break pointers.
# no args or tests/* can be passed as an argument to run all tests
pytest -s {posargs}
deps =
-rrequirements/development.txt
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test
sqlite:
SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db
SUPERSET__SQLALCHEMY_EXAMPLES_URI = sqlite:////{envtmpdir}/examples.db
mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
# docker run -p 8080:8080 --name presto starburstdata/presto
mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default
# based on https://github.com/big-data-europe/docker-hadoop
# clone the repo & run docker compose up -d to test locally
mysql-hive: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8
mysql-hive: SUPERSET__SQLALCHEMY_EXAMPLES_URI = hive://localhost:10000/default
# make sure that directory is accessible by docker
hive: UPLOAD_FOLDER = /tmp/.superset/app/static/uploads/
usedevelop = true
allowlist_externals =
npm
pkill
[testenv:cypress]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-dashboard]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh dashboard
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-explore]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh explore
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-sqllab]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh sqllab
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:cypress-sqllab-backend-persist]
setenv =
PYTHONPATH = {toxinidir}
SUPERSET_TESTENV = true
SUPERSET_CONFIG = tests.integration_tests.superset_test_config
SUPERSET_HOME = {envtmpdir}
commands =
npm install -g npm@'>=6.5.0'
pip install -e {toxinidir}/
{toxinidir}/superset-frontend/cypress_build.sh sqllab
commands_post =
pkill -if "python {envbindir}/flask"
[testenv:eslint]
changedir = {toxinidir}/superset-frontend
commands =
npm run lint
deps =
[testenv:fossa]
commands =
{toxinidir}/scripts/fossa.sh
deps =
passenv = *
[testenv:javascript]
commands =
npm install -g npm@'>=6.5.0'
{toxinidir}/superset-frontend/js_build.sh
deps =
[testenv:license-check]
commands =
{toxinidir}/scripts/check_license.sh
passenv = *
whitelist_externals =
{toxinidir}/scripts/check_license.sh
deps =
[testenv:pre-commit]
commands =
pre-commit run --all-files
deps =
-rrequirements/development.txt
skip_install = true
[testenv:pylint]
commands =
pylint superset
deps =
-rrequirements/development.txt
[testenv:thumbnails]
setenv =
SUPERSET_CONFIG = tests.integration_tests.superset_test_config_thumbnails
deps =
-rrequirements/development.txt
[tox]
envlist =
cypress-dashboard
cypress-explore
cypress-sqllab
cypress-sqllab-backend-persist
eslint
fossa
javascript
license-check
pre-commit
pylint
skipsdist = true