2019-01-15 18:53:27 -05:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
# distributed with this work for additional information
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
# under the License.
|
2018-03-21 16:13:36 -04:00
|
|
|
"""Unit tests for Superset with caching"""
|
|
|
|
import json
|
|
|
|
|
2021-01-11 08:57:55 -05:00
|
|
|
import pytest
|
|
|
|
|
2020-11-14 01:35:10 -05:00
|
|
|
from superset import app, db
|
|
|
|
from superset.extensions import cache_manager
|
2018-10-16 20:59:34 -04:00
|
|
|
from superset.utils.core import QueryStatus
|
2021-07-01 11:03:07 -04:00
|
|
|
from tests.integration_tests.fixtures.birth_names_dashboard import (
|
|
|
|
load_birth_names_dashboard_with_slices,
|
|
|
|
)
|
2019-10-18 17:44:27 -04:00
|
|
|
|
2018-03-21 16:13:36 -04:00
|
|
|
from .base_tests import SupersetTestCase
|
|
|
|
|
|
|
|
|
2020-06-29 18:36:06 -04:00
|
|
|
class TestCache(SupersetTestCase):
|
2018-03-21 16:13:36 -04:00
|
|
|
def setUp(self):
|
2020-11-14 01:35:10 -05:00
|
|
|
self.login(username="admin")
|
|
|
|
cache_manager.cache.clear()
|
|
|
|
cache_manager.data_cache.clear()
|
2018-03-21 16:13:36 -04:00
|
|
|
|
|
|
|
def tearDown(self):
|
2020-11-14 01:35:10 -05:00
|
|
|
cache_manager.cache.clear()
|
|
|
|
cache_manager.data_cache.clear()
|
|
|
|
|
2021-01-11 08:57:55 -05:00
|
|
|
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
|
2020-11-14 01:35:10 -05:00
|
|
|
def test_no_data_cache(self):
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
data_cache_config = app.config["DATA_CACHE_CONFIG"]
|
2020-11-14 01:35:10 -05:00
|
|
|
app.config["DATA_CACHE_CONFIG"] = {"CACHE_TYPE": "null"}
|
|
|
|
cache_manager.init_app(app)
|
2018-03-21 16:13:36 -04:00
|
|
|
|
2019-06-25 16:34:48 -04:00
|
|
|
slc = self.get_slice("Girls", db.session)
|
2020-11-14 01:35:10 -05:00
|
|
|
json_endpoint = "/superset/explore_json/{}/{}/".format(
|
|
|
|
slc.datasource_type, slc.datasource_id
|
|
|
|
)
|
|
|
|
resp = self.get_json_resp(
|
|
|
|
json_endpoint, {"form_data": json.dumps(slc.viz.form_data)}
|
|
|
|
)
|
|
|
|
resp_from_cache = self.get_json_resp(
|
|
|
|
json_endpoint, {"form_data": json.dumps(slc.viz.form_data)}
|
|
|
|
)
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
# restore DATA_CACHE_CONFIG
|
|
|
|
app.config["DATA_CACHE_CONFIG"] = data_cache_config
|
2020-11-14 01:35:10 -05:00
|
|
|
self.assertFalse(resp["is_cached"])
|
|
|
|
self.assertFalse(resp_from_cache["is_cached"])
|
|
|
|
|
2021-01-11 08:57:55 -05:00
|
|
|
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
|
2020-11-14 01:35:10 -05:00
|
|
|
def test_slice_data_cache(self):
|
|
|
|
# Override cache config
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
data_cache_config = app.config["DATA_CACHE_CONFIG"]
|
|
|
|
cache_default_timeout = app.config["CACHE_DEFAULT_TIMEOUT"]
|
2020-11-14 01:35:10 -05:00
|
|
|
app.config["CACHE_DEFAULT_TIMEOUT"] = 100
|
|
|
|
app.config["DATA_CACHE_CONFIG"] = {
|
|
|
|
"CACHE_TYPE": "simple",
|
|
|
|
"CACHE_DEFAULT_TIMEOUT": 10,
|
|
|
|
"CACHE_KEY_PREFIX": "superset_data_cache",
|
|
|
|
}
|
|
|
|
cache_manager.init_app(app)
|
2018-03-21 16:13:36 -04:00
|
|
|
|
2020-11-14 01:35:10 -05:00
|
|
|
slc = self.get_slice("Boys", db.session)
|
2019-06-25 16:34:48 -04:00
|
|
|
json_endpoint = "/superset/explore_json/{}/{}/".format(
|
|
|
|
slc.datasource_type, slc.datasource_id
|
2018-03-21 16:13:36 -04:00
|
|
|
)
|
|
|
|
resp = self.get_json_resp(
|
2019-06-25 16:34:48 -04:00
|
|
|
json_endpoint, {"form_data": json.dumps(slc.viz.form_data)}
|
|
|
|
)
|
2018-03-21 16:13:36 -04:00
|
|
|
resp_from_cache = self.get_json_resp(
|
2019-06-25 16:34:48 -04:00
|
|
|
json_endpoint, {"form_data": json.dumps(slc.viz.form_data)}
|
|
|
|
)
|
|
|
|
self.assertFalse(resp["is_cached"])
|
|
|
|
self.assertTrue(resp_from_cache["is_cached"])
|
2020-11-14 01:35:10 -05:00
|
|
|
# should fallback to default cache timeout
|
|
|
|
self.assertEqual(resp_from_cache["cache_timeout"], 10)
|
2019-06-25 16:34:48 -04:00
|
|
|
self.assertEqual(resp_from_cache["status"], QueryStatus.SUCCESS)
|
|
|
|
self.assertEqual(resp["data"], resp_from_cache["data"])
|
|
|
|
self.assertEqual(resp["query"], resp_from_cache["query"])
|
2020-11-14 01:35:10 -05:00
|
|
|
# should exists in `data_cache`
|
|
|
|
self.assertEqual(
|
|
|
|
cache_manager.data_cache.get(resp_from_cache["cache_key"])["query"],
|
|
|
|
resp_from_cache["query"],
|
|
|
|
)
|
|
|
|
# should not exists in `cache`
|
|
|
|
self.assertIsNone(cache_manager.cache.get(resp_from_cache["cache_key"]))
|
|
|
|
|
|
|
|
# reset cache config
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
app.config["DATA_CACHE_CONFIG"] = data_cache_config
|
|
|
|
app.config["CACHE_DEFAULT_TIMEOUT"] = cache_default_timeout
|
2020-11-14 01:35:10 -05:00
|
|
|
cache_manager.init_app(app)
|