mirror of https://github.com/apache/superset.git
chore(python-testing): move memoized tests to unit tests (#15507)
* chore: move memoized test into a separated file create integration test workflow * chore: create unit test workflow to run purely pytest * fix: bad reference * fix: remove pip requirements bc there aren't any yet * temp: install unit dependencies directly * fix: --rootdir= * fix: try to run only unit test * chore: decouple memoized as separated module * fix: bring back dependencies bc superset top-level module is coupled to flask and others so no reason no to do it * fix: reference * fix: pre-commit * fix: pylint
This commit is contained in:
parent
9ed8ce53ed
commit
62a8f2e193
|
@ -0,0 +1,193 @@
|
|||
# Python integration tests
|
||||
name: Python-Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches-ignore:
|
||||
- "dependabot/npm_and_yarn/**"
|
||||
pull_request:
|
||||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
jobs:
|
||||
test-mysql:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.7]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:5.7
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
ports:
|
||||
- 13306:3306
|
||||
redis:
|
||||
image: redis:5-alpine
|
||||
options: --entrypoint redis-server
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
apt-get-install
|
||||
pip-upgrade
|
||||
pip install -r requirements/testing.txt
|
||||
setup-mysql
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python integration tests (MySQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F mysql
|
||||
|
||||
test-postgres:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.7, 3.8]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
# Use custom ports for services to avoid accidentally connecting to
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:5-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
apt-get-install
|
||||
pip-upgrade
|
||||
pip install -r requirements/testing.txt
|
||||
setup-postgres
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python integration tests (PostgreSQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F postgres
|
||||
|
||||
test-sqlite:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.7]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||
sqlite:///${{ github.workspace }}/.temp/unittest.db
|
||||
services:
|
||||
redis:
|
||||
image: redis:5-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
apt-get-install
|
||||
pip-upgrade
|
||||
pip install -r requirements/testing.txt
|
||||
mkdir ${{ github.workspace }}/.temp
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python integration tests (SQLite)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F sqlite
|
|
@ -1,5 +1,5 @@
|
|||
# Python unit tests
|
||||
name: Python
|
||||
name: Python-Unit
|
||||
|
||||
on:
|
||||
push:
|
||||
|
@ -9,150 +9,14 @@ on:
|
|||
types: [synchronize, opened, reopened, ready_for_review]
|
||||
|
||||
jobs:
|
||||
test-mysql:
|
||||
unit-tests:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.7]
|
||||
python-version: [3.7,3.8]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||
mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
|
||||
services:
|
||||
mysql:
|
||||
image: mysql:5.7
|
||||
env:
|
||||
MYSQL_ROOT_PASSWORD: root
|
||||
ports:
|
||||
- 13306:3306
|
||||
redis:
|
||||
image: redis:5-alpine
|
||||
options: --entrypoint redis-server
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
apt-get-install
|
||||
pip-upgrade
|
||||
pip install -r requirements/testing.txt
|
||||
setup-mysql
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python unit tests (MySQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F mysql
|
||||
|
||||
test-postgres:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.7, 3.8]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
POSTGRES_USER: superset
|
||||
POSTGRES_PASSWORD: superset
|
||||
ports:
|
||||
# Use custom ports for services to avoid accidentally connecting to
|
||||
# GitHub action runner's default installations
|
||||
- 15432:5432
|
||||
redis:
|
||||
image: redis:5-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Check if python changes are present
|
||||
id: check
|
||||
env:
|
||||
GITHUB_REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
continue-on-error: true
|
||||
run: ./scripts/ci_check_no_file_changes.sh python
|
||||
- name: Setup Python
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
with:
|
||||
run: |
|
||||
apt-get-install
|
||||
pip-upgrade
|
||||
pip install -r requirements/testing.txt
|
||||
setup-postgres
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python unit tests (PostgreSQL)
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F postgres
|
||||
|
||||
test-sqlite:
|
||||
if: github.event.pull_request.draft == false
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [3.7]
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
SUPERSET_CONFIG: tests.integration_tests.superset_test_config
|
||||
REDIS_PORT: 16379
|
||||
SUPERSET__SQLALCHEMY_DATABASE_URI: |
|
||||
sqlite:///${{ github.workspace }}/.temp/unittest.db
|
||||
services:
|
||||
redis:
|
||||
image: redis:5-alpine
|
||||
ports:
|
||||
- 16379:6379
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
|
@ -171,6 +35,7 @@ jobs:
|
|||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
# TODO: separated requiermentes.txt file just for unit tests
|
||||
- name: Install dependencies
|
||||
if: steps.check.outcome == 'failure'
|
||||
uses: ./.github/actions/cached-dependencies
|
||||
|
@ -180,14 +45,11 @@ jobs:
|
|||
pip-upgrade
|
||||
pip install -r requirements/testing.txt
|
||||
mkdir ${{ github.workspace }}/.temp
|
||||
- name: Run celery
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
|
||||
- name: Python unit tests (SQLite)
|
||||
- name: Python unit tests
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
./scripts/python_tests.sh
|
||||
pytest ./tests/unit_tests --cache-clear
|
||||
- name: Upload code coverage
|
||||
if: steps.check.outcome == 'failure'
|
||||
run: |
|
||||
bash .github/workflows/codecov.sh -c -F python -F sqlite
|
||||
bash .github/workflows/codecov.sh -c -F python -F unit
|
||||
|
|
|
@ -64,6 +64,7 @@ from superset.typing import (
|
|||
)
|
||||
from superset.utils import core as utils
|
||||
from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
try:
|
||||
import requests
|
||||
|
@ -198,7 +199,7 @@ class DruidCluster(Model, AuditMixinNullable, ImportExportMixin):
|
|||
return json.loads(requests.get(endpoint, auth=auth).text)["version"]
|
||||
|
||||
@property # type: ignore
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def druid_version(self) -> str:
|
||||
return self.get_druid_version()
|
||||
|
||||
|
|
|
@ -64,6 +64,7 @@ from superset.sql_parse import ParsedQuery, Table
|
|||
from superset.utils import core as utils
|
||||
from superset.utils.core import ColumnSpec, GenericDataType
|
||||
from superset.utils.hashing import md5_sha_from_str
|
||||
from superset.utils.memoized import memoized
|
||||
from superset.utils.network import is_hostname_valid, is_port_open
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
@ -1267,7 +1268,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
|||
return parsed_query.is_select()
|
||||
|
||||
@classmethod
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def get_column_spec(
|
||||
cls,
|
||||
native_type: Optional[str],
|
||||
|
|
|
@ -38,11 +38,8 @@ from typing_extensions import TypedDict
|
|||
|
||||
from superset.exceptions import SupersetTemplateException
|
||||
from superset.extensions import feature_flag_manager
|
||||
from superset.utils.core import (
|
||||
convert_legacy_filters_into_adhoc,
|
||||
memoized,
|
||||
merge_extra_filters,
|
||||
)
|
||||
from superset.utils.core import convert_legacy_filters_into_adhoc, merge_extra_filters
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.connectors.sqla.models import SqlaTable
|
||||
|
|
|
@ -34,7 +34,7 @@ from sqlalchemy.engine.url import make_url
|
|||
from sqlalchemy.ext.declarative import declarative_base
|
||||
|
||||
from superset import db, db_engine_specs
|
||||
from superset.utils import core as utils
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
|
@ -70,7 +70,7 @@ class Slice(Base):
|
|||
datasource_id = Column(Integer)
|
||||
|
||||
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def duration_by_name(database: Database):
|
||||
return {grain.name: grain.duration for grain in database.grains()}
|
||||
|
||||
|
|
|
@ -60,6 +60,7 @@ from superset.models.helpers import AuditMixinNullable, ImportExportMixin
|
|||
from superset.models.tags import FavStarUpdater
|
||||
from superset.result_set import SupersetResultSet
|
||||
from superset.utils import cache as cache_util, core as utils
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
config = app.config
|
||||
custom_password_store = config["SQLALCHEMY_CUSTOM_PASSWORD_STORE"]
|
||||
|
@ -320,7 +321,7 @@ class Database(
|
|||
effective_username = g.user.username
|
||||
return effective_username
|
||||
|
||||
@utils.memoized(watch=("impersonate_user", "sqlalchemy_uri_decrypted", "extra"))
|
||||
@memoized(watch=("impersonate_user", "sqlalchemy_uri_decrypted", "extra"))
|
||||
def get_sqla_engine(
|
||||
self,
|
||||
schema: Optional[str] = None,
|
||||
|
@ -592,7 +593,7 @@ class Database(
|
|||
return self.get_db_engine_spec_for_backend(self.backend)
|
||||
|
||||
@classmethod
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def get_db_engine_spec_for_backend(
|
||||
cls, backend: str
|
||||
) -> Type[db_engine_specs.BaseEngineSpec]:
|
||||
|
@ -713,7 +714,7 @@ class Database(
|
|||
engine = self.get_sqla_engine()
|
||||
return engine.has_table(table_name, schema)
|
||||
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def get_dialect(self) -> Dialect:
|
||||
sqla_url = url.make_url(self.sqlalchemy_uri_decrypted)
|
||||
return sqla_url.get_dialect()() # pylint: disable=no-member
|
||||
|
|
|
@ -23,7 +23,7 @@ from sqlalchemy import Column, Integer, String
|
|||
from superset import app, db, security_manager
|
||||
from superset.connectors.connector_registry import ConnectorRegistry
|
||||
from superset.models.helpers import AuditMixinNullable
|
||||
from superset.utils import core as utils
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from superset.connectors.base.models import BaseDatasource
|
||||
|
@ -55,7 +55,7 @@ class DatasourceAccessRequest(Model, AuditMixinNullable):
|
|||
return self.get_datasource
|
||||
|
||||
@datasource.getter # type: ignore
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def get_datasource(self) -> "BaseDatasource":
|
||||
ds = db.session.query(self.cls_model).filter_by(id=self.datasource_id).first()
|
||||
return ds
|
||||
|
|
|
@ -35,6 +35,7 @@ from superset.models.tags import ChartUpdater
|
|||
from superset.tasks.thumbnails import cache_chart_thumbnail
|
||||
from superset.utils import core as utils
|
||||
from superset.utils.hashing import md5_sha_from_str
|
||||
from superset.utils.memoized import memoized
|
||||
from superset.utils.urls import get_url_path
|
||||
from superset.viz import BaseViz, viz_types # type: ignore
|
||||
|
||||
|
@ -117,7 +118,7 @@ class Slice(
|
|||
|
||||
# pylint: disable=using-constant-test
|
||||
@datasource.getter # type: ignore
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def get_datasource(self) -> Optional["BaseDatasource"]:
|
||||
return db.session.query(self.cls_model).filter_by(id=self.datasource_id).first()
|
||||
|
||||
|
@ -156,7 +157,7 @@ class Slice(
|
|||
# pylint: enable=using-constant-test
|
||||
|
||||
@property # type: ignore
|
||||
@utils.memoized
|
||||
@memoized
|
||||
def viz(self) -> Optional[BaseViz]:
|
||||
form_data = json.loads(self.params)
|
||||
viz_class = viz_types.get(self.viz_type)
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
import collections
|
||||
import decimal
|
||||
import errno
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
@ -369,65 +368,6 @@ def flasher(msg: str, severity: str = "message") -> None:
|
|||
logger.info(msg)
|
||||
|
||||
|
||||
class _memoized:
|
||||
"""Decorator that caches a function's return value each time it is called
|
||||
|
||||
If called later with the same arguments, the cached value is returned, and
|
||||
not re-evaluated.
|
||||
|
||||
Define ``watch`` as a tuple of attribute names if this Decorator
|
||||
should account for instance variable changes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, func: Callable[..., Any], watch: Optional[Tuple[str, ...]] = None
|
||||
) -> None:
|
||||
self.func = func
|
||||
self.cache: Dict[Any, Any] = {}
|
||||
self.is_method = False
|
||||
self.watch = watch or ()
|
||||
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||
key = [args, frozenset(kwargs.items())]
|
||||
if self.is_method:
|
||||
key.append(tuple([getattr(args[0], v, None) for v in self.watch]))
|
||||
key = tuple(key) # type: ignore
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
try:
|
||||
value = self.func(*args, **kwargs)
|
||||
self.cache[key] = value
|
||||
return value
|
||||
except TypeError:
|
||||
# uncachable -- for instance, passing a list as an argument.
|
||||
# Better to not cache than to blow up entirely.
|
||||
return self.func(*args, **kwargs)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Return the function's docstring."""
|
||||
return self.func.__doc__ or ""
|
||||
|
||||
def __get__(
|
||||
self, obj: Any, objtype: Type[Any]
|
||||
) -> functools.partial: # type: ignore
|
||||
if not self.is_method:
|
||||
self.is_method = True
|
||||
# Support instance methods.
|
||||
return functools.partial(self.__call__, obj)
|
||||
|
||||
|
||||
def memoized(
|
||||
func: Optional[Callable[..., Any]] = None, watch: Optional[Tuple[str, ...]] = None
|
||||
) -> Callable[..., Any]:
|
||||
if func:
|
||||
return _memoized(func)
|
||||
|
||||
def wrapper(f: Callable[..., Any]) -> Callable[..., Any]:
|
||||
return _memoized(f, watch)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def parse_js_uri_path_item(
|
||||
item: Optional[str], unquote: bool = True, eval_undefined: bool = False
|
||||
) -> Optional[str]:
|
||||
|
|
|
@ -43,7 +43,7 @@ from superset.charts.commands.exceptions import (
|
|||
TimeRangeParseFailError,
|
||||
TimeRangeUnclearError,
|
||||
)
|
||||
from superset.utils.core import memoized
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
ParserElement.enablePackrat()
|
||||
|
||||
|
|
|
@ -0,0 +1,77 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import functools
|
||||
from typing import Any, Callable, Dict, Optional, Tuple, Type
|
||||
|
||||
|
||||
class _memoized:
|
||||
"""Decorator that caches a function's return value each time it is called
|
||||
|
||||
If called later with the same arguments, the cached value is returned, and
|
||||
not re-evaluated.
|
||||
|
||||
Define ``watch`` as a tuple of attribute names if this Decorator
|
||||
should account for instance variable changes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, func: Callable[..., Any], watch: Optional[Tuple[str, ...]] = None
|
||||
) -> None:
|
||||
self.func = func
|
||||
self.cache: Dict[Any, Any] = {}
|
||||
self.is_method = False
|
||||
self.watch = watch or ()
|
||||
|
||||
def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||
key = [args, frozenset(kwargs.items())]
|
||||
if self.is_method:
|
||||
key.append(tuple([getattr(args[0], v, None) for v in self.watch]))
|
||||
key = tuple(key) # type: ignore
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
try:
|
||||
value = self.func(*args, **kwargs)
|
||||
self.cache[key] = value
|
||||
return value
|
||||
except TypeError:
|
||||
# uncachable -- for instance, passing a list as an argument.
|
||||
# Better to not cache than to blow up entirely.
|
||||
return self.func(*args, **kwargs)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Return the function's docstring."""
|
||||
return self.func.__doc__ or ""
|
||||
|
||||
def __get__(
|
||||
self, obj: Any, objtype: Type[Any]
|
||||
) -> functools.partial: # type: ignore
|
||||
if not self.is_method:
|
||||
self.is_method = True
|
||||
# Support instance methods.
|
||||
return functools.partial(self.__call__, obj)
|
||||
|
||||
|
||||
def memoized(
|
||||
func: Optional[Callable[..., Any]] = None, watch: Optional[Tuple[str, ...]] = None
|
||||
) -> Callable[..., Any]:
|
||||
if func:
|
||||
return _memoized(func)
|
||||
|
||||
def wrapper(f: Callable[..., Any]) -> Callable[..., Any]:
|
||||
return _memoized(f, watch)
|
||||
|
||||
return wrapper
|
|
@ -57,7 +57,6 @@ from superset.utils.core import (
|
|||
json_int_dttm_ser,
|
||||
json_iso_dttm_ser,
|
||||
JSONEncodedDict,
|
||||
memoized,
|
||||
merge_extra_filters,
|
||||
merge_extra_form_data,
|
||||
merge_request_params,
|
||||
|
@ -581,79 +580,6 @@ class TestUtils(SupersetTestCase):
|
|||
with self.assertRaises(SupersetException):
|
||||
validate_json(invalid)
|
||||
|
||||
def test_memoized_on_functions(self):
|
||||
watcher = {"val": 0}
|
||||
|
||||
@memoized
|
||||
def test_function(a, b, c):
|
||||
watcher["val"] += 1
|
||||
return a * b * c
|
||||
|
||||
result1 = test_function(1, 2, 3)
|
||||
result2 = test_function(1, 2, 3)
|
||||
self.assertEqual(result1, result2)
|
||||
self.assertEqual(watcher["val"], 1)
|
||||
|
||||
def test_memoized_on_methods(self):
|
||||
class test_class:
|
||||
def __init__(self, num):
|
||||
self.num = num
|
||||
self.watcher = 0
|
||||
|
||||
@memoized
|
||||
def test_method(self, a, b, c):
|
||||
self.watcher += 1
|
||||
return a * b * c * self.num
|
||||
|
||||
instance = test_class(5)
|
||||
result1 = instance.test_method(1, 2, 3)
|
||||
result2 = instance.test_method(1, 2, 3)
|
||||
self.assertEqual(result1, result2)
|
||||
self.assertEqual(instance.watcher, 1)
|
||||
instance.num = 10
|
||||
self.assertEqual(result2, instance.test_method(1, 2, 3))
|
||||
|
||||
def test_memoized_on_methods_with_watches(self):
|
||||
class test_class:
|
||||
def __init__(self, x, y):
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.watcher = 0
|
||||
|
||||
@memoized(watch=("x", "y"))
|
||||
def test_method(self, a, b, c):
|
||||
self.watcher += 1
|
||||
return a * b * c * self.x * self.y
|
||||
|
||||
instance = test_class(3, 12)
|
||||
result1 = instance.test_method(1, 2, 3)
|
||||
result2 = instance.test_method(1, 2, 3)
|
||||
self.assertEqual(result1, result2)
|
||||
self.assertEqual(instance.watcher, 1)
|
||||
result3 = instance.test_method(2, 3, 4)
|
||||
self.assertEqual(instance.watcher, 2)
|
||||
result4 = instance.test_method(2, 3, 4)
|
||||
self.assertEqual(instance.watcher, 2)
|
||||
self.assertEqual(result3, result4)
|
||||
self.assertNotEqual(result3, result1)
|
||||
instance.x = 1
|
||||
result5 = instance.test_method(2, 3, 4)
|
||||
self.assertEqual(instance.watcher, 3)
|
||||
self.assertNotEqual(result5, result4)
|
||||
result6 = instance.test_method(2, 3, 4)
|
||||
self.assertEqual(instance.watcher, 3)
|
||||
self.assertEqual(result6, result5)
|
||||
instance.x = 10
|
||||
instance.y = 10
|
||||
result7 = instance.test_method(2, 3, 4)
|
||||
self.assertEqual(instance.watcher, 4)
|
||||
self.assertNotEqual(result7, result6)
|
||||
instance.x = 3
|
||||
instance.y = 12
|
||||
result8 = instance.test_method(1, 2, 3)
|
||||
self.assertEqual(instance.watcher, 4)
|
||||
self.assertEqual(result1, result8)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_where(self):
|
||||
form_data = {"where": "a = 1"}
|
||||
|
|
|
@ -0,0 +1,96 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from pytest import mark
|
||||
|
||||
from superset.utils.memoized import memoized
|
||||
|
||||
|
||||
@mark.unittest
|
||||
class TestMemoized:
|
||||
def test_memoized_on_functions(self):
|
||||
watcher = {"val": 0}
|
||||
|
||||
@memoized
|
||||
def test_function(a, b, c):
|
||||
watcher["val"] += 1
|
||||
return a * b * c
|
||||
|
||||
result1 = test_function(1, 2, 3)
|
||||
result2 = test_function(1, 2, 3)
|
||||
assert result1 == result2
|
||||
assert watcher["val"] == 1
|
||||
|
||||
def test_memoized_on_methods(self):
|
||||
class test_class:
|
||||
def __init__(self, num):
|
||||
self.num = num
|
||||
self.watcher = 0
|
||||
|
||||
@memoized
|
||||
def test_method(self, a, b, c):
|
||||
self.watcher += 1
|
||||
return a * b * c * self.num
|
||||
|
||||
instance = test_class(5)
|
||||
result1 = instance.test_method(1, 2, 3)
|
||||
result2 = instance.test_method(1, 2, 3)
|
||||
assert result1 == result2
|
||||
assert instance.watcher == 1
|
||||
instance.num = 10
|
||||
assert result2 == instance.test_method(1, 2, 3)
|
||||
|
||||
def test_memoized_on_methods_with_watches(self):
|
||||
class test_class:
|
||||
def __init__(self, x, y):
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.watcher = 0
|
||||
|
||||
@memoized(watch=("x", "y"))
|
||||
def test_method(self, a, b, c):
|
||||
self.watcher += 1
|
||||
return a * b * c * self.x * self.y
|
||||
|
||||
instance = test_class(3, 12)
|
||||
result1 = instance.test_method(1, 2, 3)
|
||||
result2 = instance.test_method(1, 2, 3)
|
||||
assert result1 == result2
|
||||
assert instance.watcher == 1
|
||||
result3 = instance.test_method(2, 3, 4)
|
||||
assert instance.watcher == 2
|
||||
result4 = instance.test_method(2, 3, 4)
|
||||
assert instance.watcher == 2
|
||||
assert result3 == result4
|
||||
assert result3 != result1
|
||||
instance.x = 1
|
||||
result5 = instance.test_method(2, 3, 4)
|
||||
assert instance.watcher == 3
|
||||
assert result5 != result4
|
||||
result6 = instance.test_method(2, 3, 4)
|
||||
assert instance.watcher == 3
|
||||
assert result6 == result5
|
||||
instance.x = 10
|
||||
instance.y = 10
|
||||
result7 = instance.test_method(2, 3, 4)
|
||||
assert instance.watcher == 4
|
||||
assert result7 != result6
|
||||
instance.x = 3
|
||||
instance.y = 12
|
||||
result8 = instance.test_method(1, 2, 3)
|
||||
assert instance.watcher == 4
|
||||
assert result1 == result8
|
Loading…
Reference in New Issue