mirror of https://github.com/apache/superset.git
chore(python-testing): move memoized tests to unit tests (#15507)
* chore: move memoized test into a separated file create integration test workflow * chore: create unit test workflow to run purely pytest * fix: bad reference * fix: remove pip requirements bc there aren't any yet * temp: install unit dependencies directly * fix: --rootdir= * fix: try to run only unit test * chore: decouple memoized as separated module * fix: bring back dependencies bc superset top-level module is coupled to flask and others so no reason no to do it * fix: reference * fix: pre-commit * fix: pylint
This commit is contained in:
parent
9ed8ce53ed
commit
62a8f2e193
|
@ -0,0 +1,193 @@
|
||||||
|
# Python integration tests
|
||||||
|
name: Python-Integration
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches-ignore:
|
||||||
|
- "dependabot/npm_and_yarn/**"
|
||||||
|
pull_request:
|
||||||
|
types: [synchronize, opened, reopened, ready_for_review]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test-mysql:
|
||||||
|
if: github.event.pull_request.draft == false
|
||||||
|
runs-on: ubuntu-20.04
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
python-version: [3.7]
|
||||||
|
env:
|
||||||
|
PYTHONPATH: ${{ github.workspace }}
|
||||||
|
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||||
|
REDIS_PORT: 16379
|
||||||
|
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||||
|
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
||||||
|
services:
|
||||||
|
mysql:
|
||||||
|
image: mysql:5.7
|
||||||
|
env:
|
||||||
|
MYSQL_ROOT_PASSWORD: root
|
||||||
|
ports:
|
||||||
|
- 13306:3306
|
||||||
|
redis:
|
||||||
|
image: redis:5-alpine
|
||||||
|
options: --entrypoint redis-server
|
||||||
|
ports:
|
||||||
|
- 16379:6379
|
||||||
|
steps:
|
||||||
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
submodules: recursive
|
||||||
|
- name: Check if python changes are present
|
||||||
|
id: check
|
||||||
|
env:
|
||||||
|
GITHUB_REPO: ${{ github.repository }}
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
continue-on-error: true
|
||||||
|
run: ./scripts/ci_check_no_file_changes.sh python
|
||||||
|
- name: Setup Python
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
- name: Install dependencies
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
uses: ./.github/actions/cached-dependencies
|
||||||
|
with:
|
||||||
|
run: |
|
||||||
|
apt-get-install
|
||||||
|
pip-upgrade
|
||||||
|
pip install -r requirements/testing.txt
|
||||||
|
setup-mysql
|
||||||
|
- name: Run celery
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||||
|
- name: Python integration tests (MySQL)
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
./scripts/python_tests.sh
|
||||||
|
- name: Upload code coverage
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
bash .github/workflows/codecov.sh -c -F python -F mysql
|
||||||
|
|
||||||
|
test-postgres:
|
||||||
|
if: github.event.pull_request.draft == false
|
||||||
|
runs-on: ubuntu-20.04
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
python-version: [3.7, 3.8]
|
||||||
|
env:
|
||||||
|
PYTHONPATH: ${{ github.workspace }}
|
||||||
|
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||||
|
REDIS_PORT: 16379
|
||||||
|
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:10-alpine
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: superset
|
||||||
|
POSTGRES_PASSWORD: superset
|
||||||
|
ports:
|
||||||
|
# Use custom ports for services to avoid accidentally connecting to
|
||||||
|
# GitHub action runner's default installations
|
||||||
|
- 15432:5432
|
||||||
|
redis:
|
||||||
|
image: redis:5-alpine
|
||||||
|
ports:
|
||||||
|
- 16379:6379
|
||||||
|
steps:
|
||||||
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
submodules: recursive
|
||||||
|
- name: Check if python changes are present
|
||||||
|
id: check
|
||||||
|
env:
|
||||||
|
GITHUB_REPO: ${{ github.repository }}
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
continue-on-error: true
|
||||||
|
run: ./scripts/ci_check_no_file_changes.sh python
|
||||||
|
- name: Setup Python
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
- name: Install dependencies
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
uses: ./.github/actions/cached-dependencies
|
||||||
|
with:
|
||||||
|
run: |
|
||||||
|
apt-get-install
|
||||||
|
pip-upgrade
|
||||||
|
pip install -r requirements/testing.txt
|
||||||
|
setup-postgres
|
||||||
|
- name: Run celery
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||||
|
- name: Python integration tests (PostgreSQL)
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
./scripts/python_tests.sh
|
||||||
|
- name: Upload code coverage
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
bash .github/workflows/codecov.sh -c -F python -F postgres
|
||||||
|
|
||||||
|
test-sqlite:
|
||||||
|
if: github.event.pull_request.draft == false
|
||||||
|
runs-on: ubuntu-20.04
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
python-version: [3.7]
|
||||||
|
env:
|
||||||
|
PYTHONPATH: ${{ github.workspace }}
|
||||||
|
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||||
|
REDIS_PORT: 16379
|
||||||
|
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||||
|
sqlite:///${{ github.workspace }}/.temp/unittest.db
|
||||||
|
services:
|
||||||
|
redis:
|
||||||
|
image: redis:5-alpine
|
||||||
|
ports:
|
||||||
|
- 16379:6379
|
||||||
|
steps:
|
||||||
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
submodules: recursive
|
||||||
|
- name: Check if python changes are present
|
||||||
|
id: check
|
||||||
|
env:
|
||||||
|
GITHUB_REPO: ${{ github.repository }}
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
continue-on-error: true
|
||||||
|
run: ./scripts/ci_check_no_file_changes.sh python
|
||||||
|
- name: Setup Python
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
uses: actions/setup-python@v2
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.python-version }}
|
||||||
|
- name: Install dependencies
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
uses: ./.github/actions/cached-dependencies
|
||||||
|
with:
|
||||||
|
run: |
|
||||||
|
apt-get-install
|
||||||
|
pip-upgrade
|
||||||
|
pip install -r requirements/testing.txt
|
||||||
|
mkdir ${{ github.workspace }}/.temp
|
||||||
|
- name: Run celery
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||||
|
- name: Python integration tests (SQLite)
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
./scripts/python_tests.sh
|
||||||
|
- name: Upload code coverage
|
||||||
|
if: steps.check.outcome == 'failure'
|
||||||
|
run: |
|
||||||
|
bash .github/workflows/codecov.sh -c -F python -F sqlite
|
|
@ -1,5 +1,5 @@
|
||||||
# Python unit tests
|
# Python unit tests
|
||||||
name: Python
|
name: Python-Unit
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
|
@ -9,150 +9,14 @@ on:
|
||||||
types: [synchronize, opened, reopened, ready_for_review]
|
types: [synchronize, opened, reopened, ready_for_review]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test-mysql:
|
unit-tests:
|
||||||
if: github.event.pull_request.draft == false
|
if: github.event.pull_request.draft == false
|
||||||
runs-on: ubuntu-20.04
|
runs-on: ubuntu-20.04
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
python-version: [3.7]
|
python-version: [3.7,3.8]
|
||||||
env:
|
env:
|
||||||
PYTHONPATH: ${{ github.workspace }}
|
PYTHONPATH: ${{ github.workspace }}
|
||||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
|
||||||
REDIS_PORT: 16379
|
|
||||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
|
||||||
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
|
||||||
services:
|
|
||||||
mysql:
|
|
||||||
image: mysql:5.7
|
|
||||||
env:
|
|
||||||
MYSQL_ROOT_PASSWORD: root
|
|
||||||
ports:
|
|
||||||
- 13306:3306
|
|
||||||
redis:
|
|
||||||
image: redis:5-alpine
|
|
||||||
options: --entrypoint redis-server
|
|
||||||
ports:
|
|
||||||
- 16379:6379
|
|
||||||
steps:
|
|
||||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
submodules: recursive
|
|
||||||
- name: Check if python changes are present
|
|
||||||
id: check
|
|
||||||
env:
|
|
||||||
GITHUB_REPO: ${{ github.repository }}
|
|
||||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
||||||
continue-on-error: true
|
|
||||||
run: ./scripts/ci_check_no_file_changes.sh python
|
|
||||||
- name: Setup Python
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
uses: actions/setup-python@v2
|
|
||||||
with:
|
|
||||||
python-version: ${{ matrix.python-version }}
|
|
||||||
- name: Install dependencies
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
uses: ./.github/actions/cached-dependencies
|
|
||||||
with:
|
|
||||||
run: |
|
|
||||||
apt-get-install
|
|
||||||
pip-upgrade
|
|
||||||
pip install -r requirements/testing.txt
|
|
||||||
setup-mysql
|
|
||||||
- name: Run celery
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
|
||||||
- name: Python unit tests (MySQL)
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
./scripts/python_tests.sh
|
|
||||||
- name: Upload code coverage
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
bash .github/workflows/codecov.sh -c -F python -F mysql
|
|
||||||
|
|
||||||
test-postgres:
|
|
||||||
if: github.event.pull_request.draft == false
|
|
||||||
runs-on: ubuntu-20.04
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
python-version: [3.7, 3.8]
|
|
||||||
env:
|
|
||||||
PYTHONPATH: ${{ github.workspace }}
|
|
||||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
|
||||||
REDIS_PORT: 16379
|
|
||||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:10-alpine
|
|
||||||
env:
|
|
||||||
POSTGRES_USER: superset
|
|
||||||
POSTGRES_PASSWORD: superset
|
|
||||||
ports:
|
|
||||||
# Use custom ports for services to avoid accidentally connecting to
|
|
||||||
# GitHub action runner's default installations
|
|
||||||
- 15432:5432
|
|
||||||
redis:
|
|
||||||
image: redis:5-alpine
|
|
||||||
ports:
|
|
||||||
- 16379:6379
|
|
||||||
steps:
|
|
||||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
submodules: recursive
|
|
||||||
- name: Check if python changes are present
|
|
||||||
id: check
|
|
||||||
env:
|
|
||||||
GITHUB_REPO: ${{ github.repository }}
|
|
||||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
||||||
continue-on-error: true
|
|
||||||
run: ./scripts/ci_check_no_file_changes.sh python
|
|
||||||
- name: Setup Python
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
uses: actions/setup-python@v2
|
|
||||||
with:
|
|
||||||
python-version: ${{ matrix.python-version }}
|
|
||||||
- name: Install dependencies
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
uses: ./.github/actions/cached-dependencies
|
|
||||||
with:
|
|
||||||
run: |
|
|
||||||
apt-get-install
|
|
||||||
pip-upgrade
|
|
||||||
pip install -r requirements/testing.txt
|
|
||||||
setup-postgres
|
|
||||||
- name: Run celery
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
|
||||||
- name: Python unit tests (PostgreSQL)
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
./scripts/python_tests.sh
|
|
||||||
- name: Upload code coverage
|
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
bash .github/workflows/codecov.sh -c -F python -F postgres
|
|
||||||
|
|
||||||
test-sqlite:
|
|
||||||
if: github.event.pull_request.draft == false
|
|
||||||
runs-on: ubuntu-20.04
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
python-version: [3.7]
|
|
||||||
env:
|
|
||||||
PYTHONPATH: ${{ github.workspace }}
|
|
||||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
|
||||||
REDIS_PORT: 16379
|
|
||||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
|
||||||
sqlite:///${{ github.workspace }}/.temp/unittest.db
|
|
||||||
services:
|
|
||||||
redis:
|
|
||||||
image: redis:5-alpine
|
|
||||||
ports:
|
|
||||||
- 16379:6379
|
|
||||||
steps:
|
steps:
|
||||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
@ -171,6 +35,7 @@ jobs:
|
||||||
uses: actions/setup-python@v2
|
uses: actions/setup-python@v2
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
# TODO: separated requiermentes.txt file just for unit tests
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
if: steps.check.outcome == 'failure'
|
if: steps.check.outcome == 'failure'
|
||||||
uses: ./.github/actions/cached-dependencies
|
uses: ./.github/actions/cached-dependencies
|
||||||
|
@ -180,14 +45,11 @@ jobs:
|
||||||
pip-upgrade
|
pip-upgrade
|
||||||
pip install -r requirements/testing.txt
|
pip install -r requirements/testing.txt
|
||||||
mkdir ${{ github.workspace }}/.temp
|
mkdir ${{ github.workspace }}/.temp
|
||||||
- name: Run celery
|
- name: Python unit tests
|
||||||
if: steps.check.outcome == 'failure'
|
|
||||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
|
||||||
- name: Python unit tests (SQLite)
|
|
||||||
if: steps.check.outcome == 'failure'
|
if: steps.check.outcome == 'failure'
|
||||||
run: |
|
run: |
|
||||||
./scripts/python_tests.sh
|
pytest ./tests/unit_tests --cache-clear
|
||||||
- name: Upload code coverage
|
- name: Upload code coverage
|
||||||
if: steps.check.outcome == 'failure'
|
if: steps.check.outcome == 'failure'
|
||||||
run: |
|
run: |
|
||||||
bash .github/workflows/codecov.sh -c -F python -F sqlite
|
bash .github/workflows/codecov.sh -c -F python -F unit
|
||||||
|
|
|
@ -64,6 +64,7 @@ from superset.typing import (
|
||||||
)
|
)
|
||||||
from superset.utils import core as utils
|
from superset.utils import core as utils
|
||||||
from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta
|
from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta
|
||||||
|
from superset.utils.memoized import memoized
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import requests
|
import requests
|
||||||
|
@ -198,7 +199,7 @@ class DruidCluster(Model, AuditMixinNullable, ImportExportMixin):
|
||||||
return json.loads(requests.get(endpoint, auth=auth).text)["version"]
|
return json.loads(requests.get(endpoint, auth=auth).text)["version"]
|
||||||
|
|
||||||
@property # type: ignore
|
@property # type: ignore
|
||||||
@utils.memoized
|
@memoized
|
||||||
def druid_version(self) -> str:
|
def druid_version(self) -> str:
|
||||||
return self.get_druid_version()
|
return self.get_druid_version()
|
||||||
|
|
||||||
|
|
|
@ -64,6 +64,7 @@ from superset.sql_parse import ParsedQuery, Table
|
||||||
from superset.utils import core as utils
|
from superset.utils import core as utils
|
||||||
from superset.utils.core import ColumnSpec, GenericDataType
|
from superset.utils.core import ColumnSpec, GenericDataType
|
||||||
from superset.utils.hashing import md5_sha_from_str
|
from superset.utils.hashing import md5_sha_from_str
|
||||||
|
from superset.utils.memoized import memoized
|
||||||
from superset.utils.network import is_hostname_valid, is_port_open
|
from superset.utils.network import is_hostname_valid, is_port_open
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
@ -1267,7 +1268,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
||||||
return parsed_query.is_select()
|
return parsed_query.is_select()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@utils.memoized
|
@memoized
|
||||||
def get_column_spec(
|
def get_column_spec(
|
||||||
cls,
|
cls,
|
||||||
native_type: Optional[str],
|
native_type: Optional[str],
|
||||||
|
|
|
@ -38,11 +38,8 @@ from typing_extensions import TypedDict
|
||||||
|
|
||||||
from superset.exceptions import SupersetTemplateException
|
from superset.exceptions import SupersetTemplateException
|
||||||
from superset.extensions import feature_flag_manager
|
from superset.extensions import feature_flag_manager
|
||||||
from superset.utils.core import (
|
from superset.utils.core import convert_legacy_filters_into_adhoc, merge_extra_filters
|
||||||
convert_legacy_filters_into_adhoc,
|
from superset.utils.memoized import memoized
|
||||||
memoized,
|
|
||||||
merge_extra_filters,
|
|
||||||
)
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from superset.connectors.sqla.models import SqlaTable
|
from superset.connectors.sqla.models import SqlaTable
|
||||||
|
|
|
@ -34,7 +34,7 @@ from sqlalchemy.engine.url import make_url
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
|
|
||||||
from superset import db, db_engine_specs
|
from superset import db, db_engine_specs
|
||||||
from superset.utils import core as utils
|
from superset.utils.memoized import memoized
|
||||||
|
|
||||||
Base = declarative_base()
|
Base = declarative_base()
|
||||||
|
|
||||||
|
@ -70,7 +70,7 @@ class Slice(Base):
|
||||||
datasource_id = Column(Integer)
|
datasource_id = Column(Integer)
|
||||||
|
|
||||||
|
|
||||||
@utils.memoized
|
@memoized
|
||||||
def duration_by_name(database: Database):
|
def duration_by_name(database: Database):
|
||||||
return {grain.name: grain.duration for grain in database.grains()}
|
return {grain.name: grain.duration for grain in database.grains()}
|
||||||
|
|
||||||
|
|
|
@ -60,6 +60,7 @@ from superset.models.helpers import AuditMixinNullable, ImportExportMixin
|
||||||
from superset.models.tags import FavStarUpdater
|
from superset.models.tags import FavStarUpdater
|
||||||
from superset.result_set import SupersetResultSet
|
from superset.result_set import SupersetResultSet
|
||||||
from superset.utils import cache as cache_util, core as utils
|
from superset.utils import cache as cache_util, core as utils
|
||||||
|
from superset.utils.memoized import memoized
|
||||||
|
|
||||||
config = app.config
|
config = app.config
|
||||||
custom_password_store = config["SQLALCHEMY_CUSTOM_PASSWORD_STORE"]
|
custom_password_store = config["SQLALCHEMY_CUSTOM_PASSWORD_STORE"]
|
||||||
|
@ -320,7 +321,7 @@ class Database(
|
||||||
effective_username = g.user.username
|
effective_username = g.user.username
|
||||||
return effective_username
|
return effective_username
|
||||||
|
|
||||||
@utils.memoized(watch=("impersonate_user", "sqlalchemy_uri_decrypted", "extra"))
|
@memoized(watch=("impersonate_user", "sqlalchemy_uri_decrypted", "extra"))
|
||||||
def get_sqla_engine(
|
def get_sqla_engine(
|
||||||
self,
|
self,
|
||||||
schema: Optional[str] = None,
|
schema: Optional[str] = None,
|
||||||
|
@ -592,7 +593,7 @@ class Database(
|
||||||
return self.get_db_engine_spec_for_backend(self.backend)
|
return self.get_db_engine_spec_for_backend(self.backend)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@utils.memoized
|
@memoized
|
||||||
def get_db_engine_spec_for_backend(
|
def get_db_engine_spec_for_backend(
|
||||||
cls, backend: str
|
cls, backend: str
|
||||||
) -> Type[db_engine_specs.BaseEngineSpec]:
|
) -> Type[db_engine_specs.BaseEngineSpec]:
|
||||||
|
@ -713,7 +714,7 @@ class Database(
|
||||||
engine = self.get_sqla_engine()
|
engine = self.get_sqla_engine()
|
||||||
return engine.has_table(table_name, schema)
|
return engine.has_table(table_name, schema)
|
||||||
|
|
||||||
@utils.memoized
|
@memoized
|
||||||
def get_dialect(self) -> Dialect:
|
def get_dialect(self) -> Dialect:
|
||||||
sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
|
sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
|
||||||
return sqla_url.get_dialect()() # pylint: disable=no-member
|
return sqla_url.get_dialect()() # pylint: disable=no-member
|
||||||
|
|
|
@ -23,7 +23,7 @@ from sqlalchemy import Column, Integer, String
|
||||||
from superset import app, db, security_manager
|
from superset import app, db, security_manager
|
||||||
from superset.connectors.connector_registry import ConnectorRegistry
|
from superset.connectors.connector_registry import ConnectorRegistry
|
||||||
from superset.models.helpers import AuditMixinNullable
|
from superset.models.helpers import AuditMixinNullable
|
||||||
from superset.utils import core as utils
|
from superset.utils.memoized import memoized
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from superset.connectors.base.models import BaseDatasource
|
from superset.connectors.base.models import BaseDatasource
|
||||||
|
@ -55,7 +55,7 @@ class DatasourceAccessRequest(Model, AuditMixinNullable):
|
||||||
return self.get_datasource
|
return self.get_datasource
|
||||||
|
|
||||||
@datasource.getter # type: ignore
|
@datasource.getter # type: ignore
|
||||||
@utils.memoized
|
@memoized
|
||||||
def get_datasource(self) -> "BaseDatasource":
|
def get_datasource(self) -> "BaseDatasource":
|
||||||
ds = db.session.query(self.cls_model).filter_by(id=self.datasource_id).first()
|
ds = db.session.query(self.cls_model).filter_by(id=self.datasource_id).first()
|
||||||
return ds
|
return ds
|
||||||
|
|
|
@ -35,6 +35,7 @@ from superset.models.tags import ChartUpdater
|
||||||
from superset.tasks.thumbnails import cache_chart_thumbnail
|
from superset.tasks.thumbnails import cache_chart_thumbnail
|
||||||
from superset.utils import core as utils
|
from superset.utils import core as utils
|
||||||
from superset.utils.hashing import md5_sha_from_str
|
from superset.utils.hashing import md5_sha_from_str
|
||||||
|
from superset.utils.memoized import memoized
|
||||||
from superset.utils.urls import get_url_path
|
from superset.utils.urls import get_url_path
|
||||||
from superset.viz import BaseViz, viz_types # type: ignore
|
from superset.viz import BaseViz, viz_types # type: ignore
|
||||||
|
|
||||||
|
@ -117,7 +118,7 @@ class Slice(
|
||||||
|
|
||||||
# pylint: disable=using-constant-test
|
# pylint: disable=using-constant-test
|
||||||
@datasource.getter # type: ignore
|
@datasource.getter # type: ignore
|
||||||
@utils.memoized
|
@memoized
|
||||||
def get_datasource(self) -> Optional["BaseDatasource"]:
|
def get_datasource(self) -> Optional["BaseDatasource"]:
|
||||||
return db.session.query(self.cls_model).filter_by(id=self.datasource_id).first()
|
return db.session.query(self.cls_model).filter_by(id=self.datasource_id).first()
|
||||||
|
|
||||||
|
@ -156,7 +157,7 @@ class Slice(
|
||||||
# pylint: enable=using-constant-test
|
# pylint: enable=using-constant-test
|
||||||
|
|
||||||
@property # type: ignore
|
@property # type: ignore
|
||||||
@utils.memoized
|
@memoized
|
||||||
def viz(self) -> Optional[BaseViz]:
|
def viz(self) -> Optional[BaseViz]:
|
||||||
form_data = json.loads(self.params)
|
form_data = json.loads(self.params)
|
||||||
viz_class = viz_types.get(self.viz_type)
|
viz_class = viz_types.get(self.viz_type)
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
import collections
|
import collections
|
||||||
import decimal
|
import decimal
|
||||||
import errno
|
import errno
|
||||||
import functools
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
@ -369,65 +368,6 @@ def flasher(msg: str, severity: str = "message") -> None:
|
||||||
logger.info(msg)
|
logger.info(msg)
|
||||||
|
|
||||||
|
|
||||||
class _memoized:
|
|
||||||
"""Decorator that caches a function's return value each time it is called
|
|
||||||
|
|
||||||
If called later with the same arguments, the cached value is returned, and
|
|
||||||
not re-evaluated.
|
|
||||||
|
|
||||||
Define ``watch`` as a tuple of attribute names if this Decorator
|
|
||||||
should account for instance variable changes.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self, func: Callable[..., Any], watch: Optional[Tuple[str, ...]] = None
|
|
||||||
) -> None:
|
|
||||||
self.func = func
|
|
||||||
self.cache: Dict[Any, Any] = {}
|
|
||||||
self.is_method = False
|
|
||||||
self.watch = watch or ()
|
|
||||||
|
|
||||||
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
|
||||||
key = [args, frozenset(kwargs.items())]
|
|
||||||
if self.is_method:
|
|
||||||
key.append(tuple([getattr(args[0], v, None) for v in self.watch]))
|
|
||||||
key = tuple(key) # type: ignore
|
|
||||||
if key in self.cache:
|
|
||||||
return self.cache[key]
|
|
||||||
try:
|
|
||||||
value = self.func(*args, **kwargs)
|
|
||||||
self.cache[key] = value
|
|
||||||
return value
|
|
||||||
except TypeError:
|
|
||||||
# uncachable -- for instance, passing a list as an argument.
|
|
||||||
# Better to not cache than to blow up entirely.
|
|
||||||
return self.func(*args, **kwargs)
|
|
||||||
|
|
||||||
def __repr__(self) -> str:
|
|
||||||
"""Return the function's docstring."""
|
|
||||||
return self.func.__doc__ or ""
|
|
||||||
|
|
||||||
def __get__(
|
|
||||||
self, obj: Any, objtype: Type[Any]
|
|
||||||
) -> functools.partial: # type: ignore
|
|
||||||
if not self.is_method:
|
|
||||||
self.is_method = True
|
|
||||||
# Support instance methods.
|
|
||||||
return functools.partial(self.__call__, obj)
|
|
||||||
|
|
||||||
|
|
||||||
def memoized(
|
|
||||||
func: Optional[Callable[..., Any]] = None, watch: Optional[Tuple[str, ...]] = None
|
|
||||||
) -> Callable[..., Any]:
|
|
||||||
if func:
|
|
||||||
return _memoized(func)
|
|
||||||
|
|
||||||
def wrapper(f: Callable[..., Any]) -> Callable[..., Any]:
|
|
||||||
return _memoized(f, watch)
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
|
|
||||||
def parse_js_uri_path_item(
|
def parse_js_uri_path_item(
|
||||||
item: Optional[str], unquote: bool = True, eval_undefined: bool = False
|
item: Optional[str], unquote: bool = True, eval_undefined: bool = False
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
|
|
|
@ -43,7 +43,7 @@ from superset.charts.commands.exceptions import (
|
||||||
TimeRangeParseFailError,
|
TimeRangeParseFailError,
|
||||||
TimeRangeUnclearError,
|
TimeRangeUnclearError,
|
||||||
)
|
)
|
||||||
from superset.utils.core import memoized
|
from superset.utils.memoized import memoized
|
||||||
|
|
||||||
ParserElement.enablePackrat()
|
ParserElement.enablePackrat()
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,77 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
import functools
|
||||||
|
from typing import Any, Callable, Dict, Optional, Tuple, Type
|
||||||
|
|
||||||
|
|
||||||
|
class _memoized:
|
||||||
|
"""Decorator that caches a function's return value each time it is called
|
||||||
|
|
||||||
|
If called later with the same arguments, the cached value is returned, and
|
||||||
|
not re-evaluated.
|
||||||
|
|
||||||
|
Define ``watch`` as a tuple of attribute names if this Decorator
|
||||||
|
should account for instance variable changes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, func: Callable[..., Any], watch: Optional[Tuple[str, ...]] = None
|
||||||
|
) -> None:
|
||||||
|
self.func = func
|
||||||
|
self.cache: Dict[Any, Any] = {}
|
||||||
|
self.is_method = False
|
||||||
|
self.watch = watch or ()
|
||||||
|
|
||||||
|
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||||
|
key = [args, frozenset(kwargs.items())]
|
||||||
|
if self.is_method:
|
||||||
|
key.append(tuple([getattr(args[0], v, None) for v in self.watch]))
|
||||||
|
key = tuple(key) # type: ignore
|
||||||
|
if key in self.cache:
|
||||||
|
return self.cache[key]
|
||||||
|
try:
|
||||||
|
value = self.func(*args, **kwargs)
|
||||||
|
self.cache[key] = value
|
||||||
|
return value
|
||||||
|
except TypeError:
|
||||||
|
# uncachable -- for instance, passing a list as an argument.
|
||||||
|
# Better to not cache than to blow up entirely.
|
||||||
|
return self.func(*args, **kwargs)
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
"""Return the function's docstring."""
|
||||||
|
return self.func.__doc__ or ""
|
||||||
|
|
||||||
|
def __get__(
|
||||||
|
self, obj: Any, objtype: Type[Any]
|
||||||
|
) -> functools.partial: # type: ignore
|
||||||
|
if not self.is_method:
|
||||||
|
self.is_method = True
|
||||||
|
# Support instance methods.
|
||||||
|
return functools.partial(self.__call__, obj)
|
||||||
|
|
||||||
|
|
||||||
|
def memoized(
|
||||||
|
func: Optional[Callable[..., Any]] = None, watch: Optional[Tuple[str, ...]] = None
|
||||||
|
) -> Callable[..., Any]:
|
||||||
|
if func:
|
||||||
|
return _memoized(func)
|
||||||
|
|
||||||
|
def wrapper(f: Callable[..., Any]) -> Callable[..., Any]:
|
||||||
|
return _memoized(f, watch)
|
||||||
|
|
||||||
|
return wrapper
|
|
@ -57,7 +57,6 @@ from superset.utils.core import (
|
||||||
json_int_dttm_ser,
|
json_int_dttm_ser,
|
||||||
json_iso_dttm_ser,
|
json_iso_dttm_ser,
|
||||||
JSONEncodedDict,
|
JSONEncodedDict,
|
||||||
memoized,
|
|
||||||
merge_extra_filters,
|
merge_extra_filters,
|
||||||
merge_extra_form_data,
|
merge_extra_form_data,
|
||||||
merge_request_params,
|
merge_request_params,
|
||||||
|
@ -581,79 +580,6 @@ class TestUtils(SupersetTestCase):
|
||||||
with self.assertRaises(SupersetException):
|
with self.assertRaises(SupersetException):
|
||||||
validate_json(invalid)
|
validate_json(invalid)
|
||||||
|
|
||||||
def test_memoized_on_functions(self):
|
|
||||||
watcher = {"val": 0}
|
|
||||||
|
|
||||||
@memoized
|
|
||||||
def test_function(a, b, c):
|
|
||||||
watcher["val"] += 1
|
|
||||||
return a * b * c
|
|
||||||
|
|
||||||
result1 = test_function(1, 2, 3)
|
|
||||||
result2 = test_function(1, 2, 3)
|
|
||||||
self.assertEqual(result1, result2)
|
|
||||||
self.assertEqual(watcher["val"], 1)
|
|
||||||
|
|
||||||
def test_memoized_on_methods(self):
|
|
||||||
class test_class:
|
|
||||||
def __init__(self, num):
|
|
||||||
self.num = num
|
|
||||||
self.watcher = 0
|
|
||||||
|
|
||||||
@memoized
|
|
||||||
def test_method(self, a, b, c):
|
|
||||||
self.watcher += 1
|
|
||||||
return a * b * c * self.num
|
|
||||||
|
|
||||||
instance = test_class(5)
|
|
||||||
result1 = instance.test_method(1, 2, 3)
|
|
||||||
result2 = instance.test_method(1, 2, 3)
|
|
||||||
self.assertEqual(result1, result2)
|
|
||||||
self.assertEqual(instance.watcher, 1)
|
|
||||||
instance.num = 10
|
|
||||||
self.assertEqual(result2, instance.test_method(1, 2, 3))
|
|
||||||
|
|
||||||
def test_memoized_on_methods_with_watches(self):
|
|
||||||
class test_class:
|
|
||||||
def __init__(self, x, y):
|
|
||||||
self.x = x
|
|
||||||
self.y = y
|
|
||||||
self.watcher = 0
|
|
||||||
|
|
||||||
@memoized(watch=("x", "y"))
|
|
||||||
def test_method(self, a, b, c):
|
|
||||||
self.watcher += 1
|
|
||||||
return a * b * c * self.x * self.y
|
|
||||||
|
|
||||||
instance = test_class(3, 12)
|
|
||||||
result1 = instance.test_method(1, 2, 3)
|
|
||||||
result2 = instance.test_method(1, 2, 3)
|
|
||||||
self.assertEqual(result1, result2)
|
|
||||||
self.assertEqual(instance.watcher, 1)
|
|
||||||
result3 = instance.test_method(2, 3, 4)
|
|
||||||
self.assertEqual(instance.watcher, 2)
|
|
||||||
result4 = instance.test_method(2, 3, 4)
|
|
||||||
self.assertEqual(instance.watcher, 2)
|
|
||||||
self.assertEqual(result3, result4)
|
|
||||||
self.assertNotEqual(result3, result1)
|
|
||||||
instance.x = 1
|
|
||||||
result5 = instance.test_method(2, 3, 4)
|
|
||||||
self.assertEqual(instance.watcher, 3)
|
|
||||||
self.assertNotEqual(result5, result4)
|
|
||||||
result6 = instance.test_method(2, 3, 4)
|
|
||||||
self.assertEqual(instance.watcher, 3)
|
|
||||||
self.assertEqual(result6, result5)
|
|
||||||
instance.x = 10
|
|
||||||
instance.y = 10
|
|
||||||
result7 = instance.test_method(2, 3, 4)
|
|
||||||
self.assertEqual(instance.watcher, 4)
|
|
||||||
self.assertNotEqual(result7, result6)
|
|
||||||
instance.x = 3
|
|
||||||
instance.y = 12
|
|
||||||
result8 = instance.test_method(1, 2, 3)
|
|
||||||
self.assertEqual(instance.watcher, 4)
|
|
||||||
self.assertEqual(result1, result8)
|
|
||||||
|
|
||||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||||
def test_convert_legacy_filters_into_adhoc_where(self):
|
def test_convert_legacy_filters_into_adhoc_where(self):
|
||||||
form_data = {"where": "a = 1"}
|
form_data = {"where": "a = 1"}
|
||||||
|
|
|
@ -0,0 +1,96 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from pytest import mark
|
||||||
|
|
||||||
|
from superset.utils.memoized import memoized
|
||||||
|
|
||||||
|
|
||||||
|
@mark.unittest
|
||||||
|
class TestMemoized:
|
||||||
|
def test_memoized_on_functions(self):
|
||||||
|
watcher = {"val": 0}
|
||||||
|
|
||||||
|
@memoized
|
||||||
|
def test_function(a, b, c):
|
||||||
|
watcher["val"] += 1
|
||||||
|
return a * b * c
|
||||||
|
|
||||||
|
result1 = test_function(1, 2, 3)
|
||||||
|
result2 = test_function(1, 2, 3)
|
||||||
|
assert result1 == result2
|
||||||
|
assert watcher["val"] == 1
|
||||||
|
|
||||||
|
def test_memoized_on_methods(self):
|
||||||
|
class test_class:
|
||||||
|
def __init__(self, num):
|
||||||
|
self.num = num
|
||||||
|
self.watcher = 0
|
||||||
|
|
||||||
|
@memoized
|
||||||
|
def test_method(self, a, b, c):
|
||||||
|
self.watcher += 1
|
||||||
|
return a * b * c * self.num
|
||||||
|
|
||||||
|
instance = test_class(5)
|
||||||
|
result1 = instance.test_method(1, 2, 3)
|
||||||
|
result2 = instance.test_method(1, 2, 3)
|
||||||
|
assert result1 == result2
|
||||||
|
assert instance.watcher == 1
|
||||||
|
instance.num = 10
|
||||||
|
assert result2 == instance.test_method(1, 2, 3)
|
||||||
|
|
||||||
|
def test_memoized_on_methods_with_watches(self):
|
||||||
|
class test_class:
|
||||||
|
def __init__(self, x, y):
|
||||||
|
self.x = x
|
||||||
|
self.y = y
|
||||||
|
self.watcher = 0
|
||||||
|
|
||||||
|
@memoized(watch=("x", "y"))
|
||||||
|
def test_method(self, a, b, c):
|
||||||
|
self.watcher += 1
|
||||||
|
return a * b * c * self.x * self.y
|
||||||
|
|
||||||
|
instance = test_class(3, 12)
|
||||||
|
result1 = instance.test_method(1, 2, 3)
|
||||||
|
result2 = instance.test_method(1, 2, 3)
|
||||||
|
assert result1 == result2
|
||||||
|
assert instance.watcher == 1
|
||||||
|
result3 = instance.test_method(2, 3, 4)
|
||||||
|
assert instance.watcher == 2
|
||||||
|
result4 = instance.test_method(2, 3, 4)
|
||||||
|
assert instance.watcher == 2
|
||||||
|
assert result3 == result4
|
||||||
|
assert result3 != result1
|
||||||
|
instance.x = 1
|
||||||
|
result5 = instance.test_method(2, 3, 4)
|
||||||
|
assert instance.watcher == 3
|
||||||
|
assert result5 != result4
|
||||||
|
result6 = instance.test_method(2, 3, 4)
|
||||||
|
assert instance.watcher == 3
|
||||||
|
assert result6 == result5
|
||||||
|
instance.x = 10
|
||||||
|
instance.y = 10
|
||||||
|
result7 = instance.test_method(2, 3, 4)
|
||||||
|
assert instance.watcher == 4
|
||||||
|
assert result7 != result6
|
||||||
|
instance.x = 3
|
||||||
|
instance.y = 12
|
||||||
|
result8 = instance.test_method(1, 2, 3)
|
||||||
|
assert instance.watcher == 4
|
||||||
|
assert result1 == result8
|
Loading…
Reference in New Issue