2020-04-23 07:30:48 -04:00
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
# or more contributor license agreements. See the NOTICE file
|
|
|
|
# distributed with this work for additional information
|
|
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
|
|
# to you under the Apache License, Version 2.0 (the
|
|
|
|
# "License"); you may not use this file except in compliance
|
|
|
|
# with the License. You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing,
|
|
|
|
# software distributed under the License is distributed on an
|
|
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
# KIND, either express or implied. See the License for the
|
|
|
|
# specific language governing permissions and limitations
|
|
|
|
# under the License.
|
2020-12-09 15:02:29 -05:00
|
|
|
import pytest
|
|
|
|
|
2020-05-11 14:48:22 -04:00
|
|
|
import tests.test_app
|
2020-04-23 07:30:48 -04:00
|
|
|
from superset import db
|
2020-05-11 07:10:14 -04:00
|
|
|
from superset.charts.schemas import ChartDataQueryContextSchema
|
2020-04-23 07:30:48 -04:00
|
|
|
from superset.connectors.connector_registry import ConnectorRegistry
|
2020-09-01 12:41:25 -04:00
|
|
|
from superset.models.cache import CacheKey
|
2020-05-20 14:36:14 -04:00
|
|
|
from superset.utils.core import (
|
2020-08-14 13:58:24 -04:00
|
|
|
AdhocMetricExpressionType,
|
2020-06-02 03:47:28 -04:00
|
|
|
ChartDataResultFormat,
|
|
|
|
ChartDataResultType,
|
2020-08-14 13:58:24 -04:00
|
|
|
FilterOperator,
|
2020-05-20 14:36:14 -04:00
|
|
|
TimeRangeEndpoint,
|
|
|
|
)
|
2020-04-23 07:30:48 -04:00
|
|
|
from tests.base_tests import SupersetTestCase
|
2020-12-09 15:02:29 -05:00
|
|
|
from tests.fixtures.energy_dashboard import load_energy_table_with_slice
|
2020-04-23 07:30:48 -04:00
|
|
|
from tests.fixtures.query_context import get_query_context
|
|
|
|
|
|
|
|
|
2020-06-29 18:36:06 -04:00
|
|
|
class TestQueryContext(SupersetTestCase):
|
2020-05-11 07:10:14 -04:00
|
|
|
def test_schema_deserialization(self):
|
|
|
|
"""
|
|
|
|
Ensure that the deserialized QueryContext contains all required fields.
|
|
|
|
"""
|
|
|
|
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(
|
|
|
|
table.name, table.id, table.type, add_postprocessing_operations=True
|
|
|
|
)
|
2020-07-07 08:26:54 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-05-11 07:10:14 -04:00
|
|
|
self.assertEqual(len(query_context.queries), len(payload["queries"]))
|
|
|
|
for query_idx, query in enumerate(query_context.queries):
|
|
|
|
payload_query = payload["queries"][query_idx]
|
|
|
|
|
|
|
|
# check basic properies
|
|
|
|
self.assertEqual(query.extras, payload_query["extras"])
|
|
|
|
self.assertEqual(query.filter, payload_query["filters"])
|
|
|
|
self.assertEqual(query.groupby, payload_query["groupby"])
|
|
|
|
|
|
|
|
# metrics are mutated during creation
|
|
|
|
for metric_idx, metric in enumerate(query.metrics):
|
|
|
|
payload_metric = payload_query["metrics"][metric_idx]
|
|
|
|
payload_metric = (
|
|
|
|
payload_metric
|
|
|
|
if "expressionType" in payload_metric
|
|
|
|
else payload_metric["label"]
|
|
|
|
)
|
|
|
|
self.assertEqual(metric, payload_metric)
|
|
|
|
|
|
|
|
self.assertEqual(query.orderby, payload_query["orderby"])
|
|
|
|
self.assertEqual(query.time_range, payload_query["time_range"])
|
|
|
|
|
|
|
|
# check post processing operation properties
|
|
|
|
for post_proc_idx, post_proc in enumerate(query.post_processing):
|
|
|
|
payload_post_proc = payload_query["post_processing"][post_proc_idx]
|
|
|
|
self.assertEqual(post_proc["operation"], payload_post_proc["operation"])
|
|
|
|
self.assertEqual(post_proc["options"], payload_post_proc["options"])
|
|
|
|
|
2020-04-23 07:30:48 -04:00
|
|
|
def test_cache_key_changes_when_datasource_is_updated(self):
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
|
|
|
|
# construct baseline cache_key
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-04-23 07:30:48 -04:00
|
|
|
query_object = query_context.queries[0]
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
cache_key_original = query_context.query_cache_key(query_object)
|
2020-04-23 07:30:48 -04:00
|
|
|
|
|
|
|
# make temporary change and revert it to refresh the changed_on property
|
|
|
|
datasource = ConnectorRegistry.get_datasource(
|
|
|
|
datasource_type=payload["datasource"]["type"],
|
|
|
|
datasource_id=payload["datasource"]["id"],
|
2020-08-06 18:33:48 -04:00
|
|
|
session=db.session,
|
2020-04-23 07:30:48 -04:00
|
|
|
)
|
|
|
|
description_original = datasource.description
|
|
|
|
datasource.description = "temporary description"
|
|
|
|
db.session.commit()
|
|
|
|
datasource.description = description_original
|
|
|
|
db.session.commit()
|
|
|
|
|
|
|
|
# create new QueryContext with unchanged attributes and extract new cache_key
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-04-23 07:30:48 -04:00
|
|
|
query_object = query_context.queries[0]
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
cache_key_new = query_context.query_cache_key(query_object)
|
2020-04-23 07:30:48 -04:00
|
|
|
|
|
|
|
# the new cache_key should be different due to updated datasource
|
|
|
|
self.assertNotEqual(cache_key_original, cache_key_new)
|
|
|
|
|
2020-07-10 10:06:05 -04:00
|
|
|
def test_cache_key_changes_when_post_processing_is_updated(self):
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(
|
|
|
|
table.name, table.id, table.type, add_postprocessing_operations=True
|
|
|
|
)
|
|
|
|
|
|
|
|
# construct baseline cache_key from query_context with post processing operation
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-07-10 10:06:05 -04:00
|
|
|
query_object = query_context.queries[0]
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
cache_key_original = query_context.query_cache_key(query_object)
|
2020-07-10 10:06:05 -04:00
|
|
|
|
|
|
|
# ensure added None post_processing operation doesn't change cache_key
|
|
|
|
payload["queries"][0]["post_processing"].append(None)
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-07-10 10:06:05 -04:00
|
|
|
query_object = query_context.queries[0]
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
cache_key_with_null = query_context.query_cache_key(query_object)
|
2020-07-10 10:06:05 -04:00
|
|
|
self.assertEqual(cache_key_original, cache_key_with_null)
|
|
|
|
|
|
|
|
# ensure query without post processing operation is different
|
|
|
|
payload["queries"][0].pop("post_processing")
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-07-10 10:06:05 -04:00
|
|
|
query_object = query_context.queries[0]
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
cache_key_without_post_processing = query_context.query_cache_key(query_object)
|
2020-07-10 10:06:05 -04:00
|
|
|
self.assertNotEqual(cache_key_original, cache_key_without_post_processing)
|
|
|
|
|
2020-04-23 07:30:48 -04:00
|
|
|
def test_query_context_time_range_endpoints(self):
|
|
|
|
"""
|
|
|
|
Ensure that time_range_endpoints are populated automatically when missing
|
|
|
|
from the payload
|
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
del payload["queries"][0]["extras"]["time_range_endpoints"]
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-04-23 07:30:48 -04:00
|
|
|
query_object = query_context.queries[0]
|
|
|
|
extras = query_object.to_dict()["extras"]
|
|
|
|
self.assertTrue("time_range_endpoints" in extras)
|
2020-06-29 19:38:06 -04:00
|
|
|
self.assertEqual(
|
2020-04-23 07:30:48 -04:00
|
|
|
extras["time_range_endpoints"],
|
|
|
|
(TimeRangeEndpoint.INCLUSIVE, TimeRangeEndpoint.EXCLUSIVE),
|
|
|
|
)
|
|
|
|
|
|
|
|
def test_convert_deprecated_fields(self):
|
|
|
|
"""
|
|
|
|
Ensure that deprecated fields are converted correctly
|
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
payload["queries"][0]["granularity_sqla"] = "timecol"
|
2020-08-14 13:58:24 -04:00
|
|
|
payload["queries"][0]["having_filters"] = [{"col": "a", "op": "==", "val": "b"}]
|
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-04-23 07:30:48 -04:00
|
|
|
self.assertEqual(len(query_context.queries), 1)
|
|
|
|
query_object = query_context.queries[0]
|
|
|
|
self.assertEqual(query_object.granularity, "timecol")
|
|
|
|
self.assertIn("having_druid", query_object.extras)
|
2020-05-20 14:36:14 -04:00
|
|
|
|
|
|
|
def test_csv_response_format(self):
|
|
|
|
"""
|
|
|
|
Ensure that CSV result format works
|
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
2020-06-02 03:47:28 -04:00
|
|
|
payload["result_format"] = ChartDataResultFormat.CSV.value
|
2020-05-20 14:36:14 -04:00
|
|
|
payload["queries"][0]["row_limit"] = 10
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-05-20 14:36:14 -04:00
|
|
|
responses = query_context.get_payload()
|
|
|
|
self.assertEqual(len(responses), 1)
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
data = responses["queries"][0]["data"]
|
2020-05-20 14:36:14 -04:00
|
|
|
self.assertIn("name,sum__num\n", data)
|
|
|
|
self.assertEqual(len(data.split("\n")), 12)
|
|
|
|
|
2020-08-14 13:58:24 -04:00
|
|
|
def test_sql_injection_via_groupby(self):
|
|
|
|
"""
|
|
|
|
Ensure that calling invalid columns names in groupby are caught
|
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
payload["queries"][0]["groupby"] = ["currentDatabase()"]
|
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
|
|
|
query_payload = query_context.get_payload()
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
assert query_payload["queries"][0].get("error") is not None
|
2020-08-14 13:58:24 -04:00
|
|
|
|
|
|
|
def test_sql_injection_via_columns(self):
|
|
|
|
"""
|
2020-12-18 07:32:55 -05:00
|
|
|
Ensure that calling invalid column names in columns are caught
|
2020-08-14 13:58:24 -04:00
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
payload["queries"][0]["groupby"] = []
|
|
|
|
payload["queries"][0]["metrics"] = []
|
|
|
|
payload["queries"][0]["columns"] = ["*, 'extra'"]
|
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
|
|
|
query_payload = query_context.get_payload()
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
assert query_payload["queries"][0].get("error") is not None
|
2020-08-14 13:58:24 -04:00
|
|
|
|
|
|
|
def test_sql_injection_via_metrics(self):
|
|
|
|
"""
|
2020-12-18 07:32:55 -05:00
|
|
|
Ensure that calling invalid column names in filters are caught
|
2020-08-14 13:58:24 -04:00
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
payload["queries"][0]["groupby"] = ["name"]
|
|
|
|
payload["queries"][0]["metrics"] = [
|
|
|
|
{
|
|
|
|
"expressionType": AdhocMetricExpressionType.SIMPLE.value,
|
|
|
|
"column": {"column_name": "invalid_col"},
|
|
|
|
"aggregate": "SUM",
|
|
|
|
"label": "My Simple Label",
|
|
|
|
}
|
|
|
|
]
|
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
|
|
|
query_payload = query_context.get_payload()
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
assert query_payload["queries"][0].get("error") is not None
|
2020-08-14 13:58:24 -04:00
|
|
|
|
2020-05-20 14:36:14 -04:00
|
|
|
def test_samples_response_type(self):
|
|
|
|
"""
|
|
|
|
Ensure that samples result type works
|
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
2020-06-02 03:47:28 -04:00
|
|
|
payload["result_type"] = ChartDataResultType.SAMPLES.value
|
2020-05-20 14:36:14 -04:00
|
|
|
payload["queries"][0]["row_limit"] = 5
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-05-20 14:36:14 -04:00
|
|
|
responses = query_context.get_payload()
|
|
|
|
self.assertEqual(len(responses), 1)
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
data = responses["queries"][0]["data"]
|
2020-05-20 14:36:14 -04:00
|
|
|
self.assertIsInstance(data, list)
|
|
|
|
self.assertEqual(len(data), 5)
|
|
|
|
self.assertNotIn("sum__num", data[0])
|
|
|
|
|
|
|
|
def test_query_response_type(self):
|
|
|
|
"""
|
|
|
|
Ensure that query result type works
|
|
|
|
"""
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
2020-06-02 03:47:28 -04:00
|
|
|
payload["result_type"] = ChartDataResultType.QUERY.value
|
2020-08-14 13:58:24 -04:00
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
2020-05-20 14:36:14 -04:00
|
|
|
responses = query_context.get_payload()
|
|
|
|
self.assertEqual(len(responses), 1)
|
feat(SIP-39): Async query support for charts (#11499)
* Generate JWT in Flask app
* Refactor chart data API query logic, add JWT validation and async worker
* Add redis stream implementation, refactoring
* Add chart data cache endpoint, refactor QueryContext caching
* Typing, linting, refactoring
* pytest fixes and openapi schema update
* Enforce caching be configured for async query init
* Async query processing for explore_json endpoint
* Add /api/v1/async_event endpoint
* Async frontend for dashboards [WIP]
* Chart async error message support, refactoring
* Abstract asyncEvent middleware
* Async chart loading for Explore
* Pylint fixes
* asyncEvent middleware -> TypeScript, JS linting
* Chart data API: enforce forced_cache, add tests
* Add tests for explore_json endpoints
* Add test for chart data cache enpoint (no login)
* Consolidate set_and_log_cache and add STORE_CACHE_KEYS_IN_METADATA_DB flag
* Add tests for tasks/async_queries and address PR comments
* Bypass non-JSON result formats for async queries
* Add tests for redux middleware
* Remove debug statement
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
* Skip force_cached if no queryObj
* SunburstViz: don't modify self.form_data
* Fix failing annotation test
* Resolve merge/lint issues
* Reduce polling delay
* Fix new getClientErrorObject reference
* Fix flakey unit tests
* /api/v1/async_event: increment redis stream ID, add tests
* PR feedback: refactoring, configuration
* Fixup: remove debugging
* Fix typescript errors due to redux upgrade
* Update UPDATING.md
* Fix failing py tests
* asyncEvent_spec.js -> asyncEvent_spec.ts
* Refactor flakey Python 3.7 mock assertions
* Fix another shared state issue in Py tests
* Use 'sub' claim in JWT for user_id
* Refactor async middleware config
* Fixup: restore FeatureFlag boolean type
Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com>
2020-12-10 23:21:56 -05:00
|
|
|
response = responses["queries"][0]
|
2020-05-20 14:36:14 -04:00
|
|
|
self.assertEqual(len(response), 2)
|
|
|
|
self.assertEqual(response["language"], "sql")
|
|
|
|
self.assertIn("SELECT", response["query"])
|
2020-12-18 07:32:55 -05:00
|
|
|
|
|
|
|
def test_query_object_unknown_fields(self):
|
|
|
|
"""
|
|
|
|
Ensure that query objects with unknown fields don't raise an Exception and
|
|
|
|
have an identical cache key as one without the unknown field
|
|
|
|
"""
|
|
|
|
self.maxDiff = None
|
|
|
|
self.login(username="admin")
|
|
|
|
table_name = "birth_names"
|
|
|
|
table = self.get_table_by_name(table_name)
|
|
|
|
payload = get_query_context(table.name, table.id, table.type)
|
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
|
|
|
responses = query_context.get_payload()
|
|
|
|
orig_cache_key = responses["queries"][0]["cache_key"]
|
|
|
|
payload["queries"][0]["foo"] = "bar"
|
|
|
|
query_context = ChartDataQueryContextSchema().load(payload)
|
|
|
|
responses = query_context.get_payload()
|
|
|
|
new_cache_key = responses["queries"][0]["cache_key"]
|
|
|
|
self.assertEqual(orig_cache_key, new_cache_key)
|