From 62a8f2e193da3bdf1f43be085b5814be3dd476fa Mon Sep 17 00:00:00 2001 From: Amit Miran <47772523+amitmiran137@users.noreply.github.com> Date: Mon, 12 Jul 2021 10:00:18 +0300 Subject: [PATCH] chore(python-testing): move memoized tests to unit tests (#15507) * chore: move memoized test into a separated file create integration test workflow * chore: create unit test workflow to run purely pytest * fix: bad reference * fix: remove pip requirements bc there aren't any yet * temp: install unit dependencies directly * fix: --rootdir= * fix: try to run only unit test * chore: decouple memoized as separated module * fix: bring back dependencies bc superset top-level module is coupled to flask and others so no reason no to do it * fix: reference * fix: pre-commit * fix: pylint --- .../superset-python-integrationtest.yml | 193 ++++++++++++++++++ .../workflows/superset-python-unittest.yml | 152 +------------- superset/connectors/druid/models.py | 3 +- superset/db_engine_specs/base.py | 3 +- superset/jinja_context.py | 7 +- .../620241d1153f_update_time_grain_sqla.py | 4 +- superset/models/core.py | 7 +- superset/models/datasource_access_request.py | 4 +- superset/models/slice.py | 5 +- superset/utils/core.py | 60 ------ superset/utils/date_parser.py | 2 +- superset/utils/memoized.py | 77 +++++++ tests/integration_tests/utils_tests.py | 74 ------- tests/unit_tests/memoized_tests.py | 96 +++++++++ 14 files changed, 391 insertions(+), 296 deletions(-) create mode 100644 .github/workflows/superset-python-integrationtest.yml create mode 100644 superset/utils/memoized.py create mode 100644 tests/unit_tests/memoized_tests.py diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml new file mode 100644 index 0000000000..c006d22424 --- /dev/null +++ b/.github/workflows/superset-python-integrationtest.yml @@ -0,0 +1,193 @@ +# Python integration tests +name: Python-Integration + +on: + push: + branches-ignore: + - "dependabot/npm_and_yarn/**" + pull_request: + types: [synchronize, opened, reopened, ready_for_review] + +jobs: + test-mysql: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.7] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.integration_tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: | + mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 13306:3306 + redis: + image: redis:5-alpine + options: --entrypoint redis-server + ports: + - 16379:6379 + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python + - name: Setup Python + if: steps.check.outcome == 'failure' + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + if: steps.check.outcome == 'failure' + uses: ./.github/actions/cached-dependencies + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-mysql + - name: Run celery + if: steps.check.outcome == 'failure' + run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & + - name: Python integration tests (MySQL) + if: steps.check.outcome == 'failure' + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + if: steps.check.outcome == 'failure' + run: | + bash .github/workflows/codecov.sh -c -F python -F mysql + + test-postgres: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.7, 3.8] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.integration_tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset + services: + postgres: + image: postgres:10-alpine + env: + POSTGRES_USER: superset + POSTGRES_PASSWORD: superset + ports: + # Use custom ports for services to avoid accidentally connecting to + # GitHub action runner's default installations + - 15432:5432 + redis: + image: redis:5-alpine + ports: + - 16379:6379 + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python + - name: Setup Python + if: steps.check.outcome == 'failure' + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + if: steps.check.outcome == 'failure' + uses: ./.github/actions/cached-dependencies + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + setup-postgres + - name: Run celery + if: steps.check.outcome == 'failure' + run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & + - name: Python integration tests (PostgreSQL) + if: steps.check.outcome == 'failure' + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + if: steps.check.outcome == 'failure' + run: | + bash .github/workflows/codecov.sh -c -F python -F postgres + + test-sqlite: + if: github.event.pull_request.draft == false + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.7] + env: + PYTHONPATH: ${{ github.workspace }} + SUPERSET_CONFIG: tests.integration_tests.superset_test_config + REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: | + sqlite:///${{ github.workspace }}/.temp/unittest.db + services: + redis: + image: redis:5-alpine + ports: + - 16379:6379 + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v2 + with: + persist-credentials: false + submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python + - name: Setup Python + if: steps.check.outcome == 'failure' + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + if: steps.check.outcome == 'failure' + uses: ./.github/actions/cached-dependencies + with: + run: | + apt-get-install + pip-upgrade + pip install -r requirements/testing.txt + mkdir ${{ github.workspace }}/.temp + - name: Run celery + if: steps.check.outcome == 'failure' + run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & + - name: Python integration tests (SQLite) + if: steps.check.outcome == 'failure' + run: | + ./scripts/python_tests.sh + - name: Upload code coverage + if: steps.check.outcome == 'failure' + run: | + bash .github/workflows/codecov.sh -c -F python -F sqlite diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 0ff0266ae1..69f96bcb9b 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -1,5 +1,5 @@ # Python unit tests -name: Python +name: Python-Unit on: push: @@ -9,150 +9,14 @@ on: types: [synchronize, opened, reopened, ready_for_review] jobs: - test-mysql: + unit-tests: if: github.event.pull_request.draft == false runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.7] + python-version: [3.7,3.8] env: PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.integration_tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: | - mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true - services: - mysql: - image: mysql:5.7 - env: - MYSQL_ROOT_PASSWORD: root - ports: - - 13306:3306 - redis: - image: redis:5-alpine - options: --entrypoint redis-server - ports: - - 16379:6379 - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Check if python changes are present - id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python - - name: Setup Python - if: steps.check.outcome == 'failure' - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - if: steps.check.outcome == 'failure' - uses: ./.github/actions/cached-dependencies - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - setup-mysql - - name: Run celery - if: steps.check.outcome == 'failure' - run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - - name: Python unit tests (MySQL) - if: steps.check.outcome == 'failure' - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - if: steps.check.outcome == 'failure' - run: | - bash .github/workflows/codecov.sh -c -F python -F mysql - - test-postgres: - if: github.event.pull_request.draft == false - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.7, 3.8] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.integration_tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset - services: - postgres: - image: postgres:10-alpine - env: - POSTGRES_USER: superset - POSTGRES_PASSWORD: superset - ports: - # Use custom ports for services to avoid accidentally connecting to - # GitHub action runner's default installations - - 15432:5432 - redis: - image: redis:5-alpine - ports: - - 16379:6379 - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: Check if python changes are present - id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python - - name: Setup Python - if: steps.check.outcome == 'failure' - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - if: steps.check.outcome == 'failure' - uses: ./.github/actions/cached-dependencies - with: - run: | - apt-get-install - pip-upgrade - pip install -r requirements/testing.txt - setup-postgres - - name: Run celery - if: steps.check.outcome == 'failure' - run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - - name: Python unit tests (PostgreSQL) - if: steps.check.outcome == 'failure' - run: | - ./scripts/python_tests.sh - - name: Upload code coverage - if: steps.check.outcome == 'failure' - run: | - bash .github/workflows/codecov.sh -c -F python -F postgres - - test-sqlite: - if: github.event.pull_request.draft == false - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.7] - env: - PYTHONPATH: ${{ github.workspace }} - SUPERSET_CONFIG: tests.integration_tests.superset_test_config - REDIS_PORT: 16379 - SUPERSET__SQLALCHEMY_DATABASE_URI: | - sqlite:///${{ github.workspace }}/.temp/unittest.db - services: - redis: - image: redis:5-alpine - ports: - - 16379:6379 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@v2 @@ -171,6 +35,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} +# TODO: separated requiermentes.txt file just for unit tests - name: Install dependencies if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies @@ -180,14 +45,11 @@ jobs: pip-upgrade pip install -r requirements/testing.txt mkdir ${{ github.workspace }}/.temp - - name: Run celery - if: steps.check.outcome == 'failure' - run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - - name: Python unit tests (SQLite) + - name: Python unit tests if: steps.check.outcome == 'failure' run: | - ./scripts/python_tests.sh + pytest ./tests/unit_tests --cache-clear - name: Upload code coverage if: steps.check.outcome == 'failure' run: | - bash .github/workflows/codecov.sh -c -F python -F sqlite + bash .github/workflows/codecov.sh -c -F python -F unit diff --git a/superset/connectors/druid/models.py b/superset/connectors/druid/models.py index 14eda04673..03c2ccb9ba 100644 --- a/superset/connectors/druid/models.py +++ b/superset/connectors/druid/models.py @@ -64,6 +64,7 @@ from superset.typing import ( ) from superset.utils import core as utils from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta +from superset.utils.memoized import memoized try: import requests @@ -198,7 +199,7 @@ class DruidCluster(Model, AuditMixinNullable, ImportExportMixin): return json.loads(requests.get(endpoint, auth=auth).text)["version"] @property # type: ignore - @utils.memoized + @memoized def druid_version(self) -> str: return self.get_druid_version() diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py index 4691f2be55..7cb8e112d6 100644 --- a/superset/db_engine_specs/base.py +++ b/superset/db_engine_specs/base.py @@ -64,6 +64,7 @@ from superset.sql_parse import ParsedQuery, Table from superset.utils import core as utils from superset.utils.core import ColumnSpec, GenericDataType from superset.utils.hashing import md5_sha_from_str +from superset.utils.memoized import memoized from superset.utils.network import is_hostname_valid, is_port_open if TYPE_CHECKING: @@ -1267,7 +1268,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods return parsed_query.is_select() @classmethod - @utils.memoized + @memoized def get_column_spec( cls, native_type: Optional[str], diff --git a/superset/jinja_context.py b/superset/jinja_context.py index 43e64e4be7..6938c478fd 100644 --- a/superset/jinja_context.py +++ b/superset/jinja_context.py @@ -38,11 +38,8 @@ from typing_extensions import TypedDict from superset.exceptions import SupersetTemplateException from superset.extensions import feature_flag_manager -from superset.utils.core import ( - convert_legacy_filters_into_adhoc, - memoized, - merge_extra_filters, -) +from superset.utils.core import convert_legacy_filters_into_adhoc, merge_extra_filters +from superset.utils.memoized import memoized if TYPE_CHECKING: from superset.connectors.sqla.models import SqlaTable diff --git a/superset/migrations/versions/620241d1153f_update_time_grain_sqla.py b/superset/migrations/versions/620241d1153f_update_time_grain_sqla.py index c5c779bc73..560b6106f4 100644 --- a/superset/migrations/versions/620241d1153f_update_time_grain_sqla.py +++ b/superset/migrations/versions/620241d1153f_update_time_grain_sqla.py @@ -34,7 +34,7 @@ from sqlalchemy.engine.url import make_url from sqlalchemy.ext.declarative import declarative_base from superset import db, db_engine_specs -from superset.utils import core as utils +from superset.utils.memoized import memoized Base = declarative_base() @@ -70,7 +70,7 @@ class Slice(Base): datasource_id = Column(Integer) -@utils.memoized +@memoized def duration_by_name(database: Database): return {grain.name: grain.duration for grain in database.grains()} diff --git a/superset/models/core.py b/superset/models/core.py index 4173377e06..20ba0e5a75 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -60,6 +60,7 @@ from superset.models.helpers import AuditMixinNullable, ImportExportMixin from superset.models.tags import FavStarUpdater from superset.result_set import SupersetResultSet from superset.utils import cache as cache_util, core as utils +from superset.utils.memoized import memoized config = app.config custom_password_store = config["SQLALCHEMY_CUSTOM_PASSWORD_STORE"] @@ -320,7 +321,7 @@ class Database( effective_username = g.user.username return effective_username - @utils.memoized(watch=("impersonate_user", "sqlalchemy_uri_decrypted", "extra")) + @memoized(watch=("impersonate_user", "sqlalchemy_uri_decrypted", "extra")) def get_sqla_engine( self, schema: Optional[str] = None, @@ -592,7 +593,7 @@ class Database( return self.get_db_engine_spec_for_backend(self.backend) @classmethod - @utils.memoized + @memoized def get_db_engine_spec_for_backend( cls, backend: str ) -> Type[db_engine_specs.BaseEngineSpec]: @@ -713,7 +714,7 @@ class Database( engine = self.get_sqla_engine() return engine.has_table(table_name, schema) - @utils.memoized + @memoized def get_dialect(self) -> Dialect: sqla_url = url.make_url(self.sqlalchemy_uri_decrypted) return sqla_url.get_dialect()() # pylint: disable=no-member diff --git a/superset/models/datasource_access_request.py b/superset/models/datasource_access_request.py index 98362ec316..fa3b9d6711 100644 --- a/superset/models/datasource_access_request.py +++ b/superset/models/datasource_access_request.py @@ -23,7 +23,7 @@ from sqlalchemy import Column, Integer, String from superset import app, db, security_manager from superset.connectors.connector_registry import ConnectorRegistry from superset.models.helpers import AuditMixinNullable -from superset.utils import core as utils +from superset.utils.memoized import memoized if TYPE_CHECKING: from superset.connectors.base.models import BaseDatasource @@ -55,7 +55,7 @@ class DatasourceAccessRequest(Model, AuditMixinNullable): return self.get_datasource @datasource.getter # type: ignore - @utils.memoized + @memoized def get_datasource(self) -> "BaseDatasource": ds = db.session.query(self.cls_model).filter_by(id=self.datasource_id).first() return ds diff --git a/superset/models/slice.py b/superset/models/slice.py index c80e1fba78..676c8653d2 100644 --- a/superset/models/slice.py +++ b/superset/models/slice.py @@ -35,6 +35,7 @@ from superset.models.tags import ChartUpdater from superset.tasks.thumbnails import cache_chart_thumbnail from superset.utils import core as utils from superset.utils.hashing import md5_sha_from_str +from superset.utils.memoized import memoized from superset.utils.urls import get_url_path from superset.viz import BaseViz, viz_types # type: ignore @@ -117,7 +118,7 @@ class Slice( # pylint: disable=using-constant-test @datasource.getter # type: ignore - @utils.memoized + @memoized def get_datasource(self) -> Optional["BaseDatasource"]: return db.session.query(self.cls_model).filter_by(id=self.datasource_id).first() @@ -156,7 +157,7 @@ class Slice( # pylint: enable=using-constant-test @property # type: ignore - @utils.memoized + @memoized def viz(self) -> Optional[BaseViz]: form_data = json.loads(self.params) viz_class = viz_types.get(self.viz_type) diff --git a/superset/utils/core.py b/superset/utils/core.py index 50dc87e92c..936e191bc9 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -18,7 +18,6 @@ import collections import decimal import errno -import functools import json import logging import os @@ -369,65 +368,6 @@ def flasher(msg: str, severity: str = "message") -> None: logger.info(msg) -class _memoized: - """Decorator that caches a function's return value each time it is called - - If called later with the same arguments, the cached value is returned, and - not re-evaluated. - - Define ``watch`` as a tuple of attribute names if this Decorator - should account for instance variable changes. - """ - - def __init__( - self, func: Callable[..., Any], watch: Optional[Tuple[str, ...]] = None - ) -> None: - self.func = func - self.cache: Dict[Any, Any] = {} - self.is_method = False - self.watch = watch or () - - def __call__(self, *args: Any, **kwargs: Any) -> Any: - key = [args, frozenset(kwargs.items())] - if self.is_method: - key.append(tuple([getattr(args[0], v, None) for v in self.watch])) - key = tuple(key) # type: ignore - if key in self.cache: - return self.cache[key] - try: - value = self.func(*args, **kwargs) - self.cache[key] = value - return value - except TypeError: - # uncachable -- for instance, passing a list as an argument. - # Better to not cache than to blow up entirely. - return self.func(*args, **kwargs) - - def __repr__(self) -> str: - """Return the function's docstring.""" - return self.func.__doc__ or "" - - def __get__( - self, obj: Any, objtype: Type[Any] - ) -> functools.partial: # type: ignore - if not self.is_method: - self.is_method = True - # Support instance methods. - return functools.partial(self.__call__, obj) - - -def memoized( - func: Optional[Callable[..., Any]] = None, watch: Optional[Tuple[str, ...]] = None -) -> Callable[..., Any]: - if func: - return _memoized(func) - - def wrapper(f: Callable[..., Any]) -> Callable[..., Any]: - return _memoized(f, watch) - - return wrapper - - def parse_js_uri_path_item( item: Optional[str], unquote: bool = True, eval_undefined: bool = False ) -> Optional[str]: diff --git a/superset/utils/date_parser.py b/superset/utils/date_parser.py index 42625e66a2..9bdf1d3026 100644 --- a/superset/utils/date_parser.py +++ b/superset/utils/date_parser.py @@ -43,7 +43,7 @@ from superset.charts.commands.exceptions import ( TimeRangeParseFailError, TimeRangeUnclearError, ) -from superset.utils.core import memoized +from superset.utils.memoized import memoized ParserElement.enablePackrat() diff --git a/superset/utils/memoized.py b/superset/utils/memoized.py new file mode 100644 index 0000000000..ca496bd937 --- /dev/null +++ b/superset/utils/memoized.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import functools +from typing import Any, Callable, Dict, Optional, Tuple, Type + + +class _memoized: + """Decorator that caches a function's return value each time it is called + + If called later with the same arguments, the cached value is returned, and + not re-evaluated. + + Define ``watch`` as a tuple of attribute names if this Decorator + should account for instance variable changes. + """ + + def __init__( + self, func: Callable[..., Any], watch: Optional[Tuple[str, ...]] = None + ) -> None: + self.func = func + self.cache: Dict[Any, Any] = {} + self.is_method = False + self.watch = watch or () + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + key = [args, frozenset(kwargs.items())] + if self.is_method: + key.append(tuple([getattr(args[0], v, None) for v in self.watch])) + key = tuple(key) # type: ignore + if key in self.cache: + return self.cache[key] + try: + value = self.func(*args, **kwargs) + self.cache[key] = value + return value + except TypeError: + # uncachable -- for instance, passing a list as an argument. + # Better to not cache than to blow up entirely. + return self.func(*args, **kwargs) + + def __repr__(self) -> str: + """Return the function's docstring.""" + return self.func.__doc__ or "" + + def __get__( + self, obj: Any, objtype: Type[Any] + ) -> functools.partial: # type: ignore + if not self.is_method: + self.is_method = True + # Support instance methods. + return functools.partial(self.__call__, obj) + + +def memoized( + func: Optional[Callable[..., Any]] = None, watch: Optional[Tuple[str, ...]] = None +) -> Callable[..., Any]: + if func: + return _memoized(func) + + def wrapper(f: Callable[..., Any]) -> Callable[..., Any]: + return _memoized(f, watch) + + return wrapper diff --git a/tests/integration_tests/utils_tests.py b/tests/integration_tests/utils_tests.py index 86b5957fd5..2d668329d3 100644 --- a/tests/integration_tests/utils_tests.py +++ b/tests/integration_tests/utils_tests.py @@ -57,7 +57,6 @@ from superset.utils.core import ( json_int_dttm_ser, json_iso_dttm_ser, JSONEncodedDict, - memoized, merge_extra_filters, merge_extra_form_data, merge_request_params, @@ -581,79 +580,6 @@ class TestUtils(SupersetTestCase): with self.assertRaises(SupersetException): validate_json(invalid) - def test_memoized_on_functions(self): - watcher = {"val": 0} - - @memoized - def test_function(a, b, c): - watcher["val"] += 1 - return a * b * c - - result1 = test_function(1, 2, 3) - result2 = test_function(1, 2, 3) - self.assertEqual(result1, result2) - self.assertEqual(watcher["val"], 1) - - def test_memoized_on_methods(self): - class test_class: - def __init__(self, num): - self.num = num - self.watcher = 0 - - @memoized - def test_method(self, a, b, c): - self.watcher += 1 - return a * b * c * self.num - - instance = test_class(5) - result1 = instance.test_method(1, 2, 3) - result2 = instance.test_method(1, 2, 3) - self.assertEqual(result1, result2) - self.assertEqual(instance.watcher, 1) - instance.num = 10 - self.assertEqual(result2, instance.test_method(1, 2, 3)) - - def test_memoized_on_methods_with_watches(self): - class test_class: - def __init__(self, x, y): - self.x = x - self.y = y - self.watcher = 0 - - @memoized(watch=("x", "y")) - def test_method(self, a, b, c): - self.watcher += 1 - return a * b * c * self.x * self.y - - instance = test_class(3, 12) - result1 = instance.test_method(1, 2, 3) - result2 = instance.test_method(1, 2, 3) - self.assertEqual(result1, result2) - self.assertEqual(instance.watcher, 1) - result3 = instance.test_method(2, 3, 4) - self.assertEqual(instance.watcher, 2) - result4 = instance.test_method(2, 3, 4) - self.assertEqual(instance.watcher, 2) - self.assertEqual(result3, result4) - self.assertNotEqual(result3, result1) - instance.x = 1 - result5 = instance.test_method(2, 3, 4) - self.assertEqual(instance.watcher, 3) - self.assertNotEqual(result5, result4) - result6 = instance.test_method(2, 3, 4) - self.assertEqual(instance.watcher, 3) - self.assertEqual(result6, result5) - instance.x = 10 - instance.y = 10 - result7 = instance.test_method(2, 3, 4) - self.assertEqual(instance.watcher, 4) - self.assertNotEqual(result7, result6) - instance.x = 3 - instance.y = 12 - result8 = instance.test_method(1, 2, 3) - self.assertEqual(instance.watcher, 4) - self.assertEqual(result1, result8) - @patch("superset.utils.core.to_adhoc", mock_to_adhoc) def test_convert_legacy_filters_into_adhoc_where(self): form_data = {"where": "a = 1"} diff --git a/tests/unit_tests/memoized_tests.py b/tests/unit_tests/memoized_tests.py new file mode 100644 index 0000000000..3b3f436606 --- /dev/null +++ b/tests/unit_tests/memoized_tests.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pytest import mark + +from superset.utils.memoized import memoized + + +@mark.unittest +class TestMemoized: + def test_memoized_on_functions(self): + watcher = {"val": 0} + + @memoized + def test_function(a, b, c): + watcher["val"] += 1 + return a * b * c + + result1 = test_function(1, 2, 3) + result2 = test_function(1, 2, 3) + assert result1 == result2 + assert watcher["val"] == 1 + + def test_memoized_on_methods(self): + class test_class: + def __init__(self, num): + self.num = num + self.watcher = 0 + + @memoized + def test_method(self, a, b, c): + self.watcher += 1 + return a * b * c * self.num + + instance = test_class(5) + result1 = instance.test_method(1, 2, 3) + result2 = instance.test_method(1, 2, 3) + assert result1 == result2 + assert instance.watcher == 1 + instance.num = 10 + assert result2 == instance.test_method(1, 2, 3) + + def test_memoized_on_methods_with_watches(self): + class test_class: + def __init__(self, x, y): + self.x = x + self.y = y + self.watcher = 0 + + @memoized(watch=("x", "y")) + def test_method(self, a, b, c): + self.watcher += 1 + return a * b * c * self.x * self.y + + instance = test_class(3, 12) + result1 = instance.test_method(1, 2, 3) + result2 = instance.test_method(1, 2, 3) + assert result1 == result2 + assert instance.watcher == 1 + result3 = instance.test_method(2, 3, 4) + assert instance.watcher == 2 + result4 = instance.test_method(2, 3, 4) + assert instance.watcher == 2 + assert result3 == result4 + assert result3 != result1 + instance.x = 1 + result5 = instance.test_method(2, 3, 4) + assert instance.watcher == 3 + assert result5 != result4 + result6 = instance.test_method(2, 3, 4) + assert instance.watcher == 3 + assert result6 == result5 + instance.x = 10 + instance.y = 10 + result7 = instance.test_method(2, 3, 4) + assert instance.watcher == 4 + assert result7 != result6 + instance.x = 3 + instance.y = 12 + result8 = instance.test_method(1, 2, 3) + assert instance.watcher == 4 + assert result1 == result8