mirror of https://github.com/apache/superset.git
941 lines
39 KiB
Python
941 lines
39 KiB
Python
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
import json
|
|
import unittest
|
|
from unittest.mock import Mock
|
|
|
|
from pydruid.utils.dimensions import MapLookupExtraction, RegexExtraction
|
|
import pydruid.utils.postaggregator as postaggs
|
|
|
|
|
|
import superset.connectors.druid.models as models
|
|
from superset.connectors.druid.models import (
|
|
DruidColumn, DruidDatasource, DruidMetric,
|
|
)
|
|
from superset.exceptions import SupersetException
|
|
|
|
|
|
def mock_metric(metric_name, is_postagg=False):
|
|
metric = Mock()
|
|
metric.metric_name = metric_name
|
|
metric.metric_type = 'postagg' if is_postagg else 'metric'
|
|
return metric
|
|
|
|
|
|
def emplace(metrics_dict, metric_name, is_postagg=False):
|
|
metrics_dict[metric_name] = mock_metric(metric_name, is_postagg)
|
|
|
|
|
|
# Unit tests that can be run without initializing base tests
|
|
class DruidFuncTestCase(unittest.TestCase):
|
|
|
|
def test_get_filters_extraction_fn_map(self):
|
|
filters = [{'col': 'deviceName', 'val': ['iPhone X'], 'op': 'in'}]
|
|
dimension_spec = {
|
|
'type': 'extraction',
|
|
'dimension': 'device',
|
|
'outputName': 'deviceName',
|
|
'outputType': 'STRING',
|
|
'extractionFn': {
|
|
'type': 'lookup',
|
|
'dimension': 'dimensionName',
|
|
'outputName': 'dimensionOutputName',
|
|
'replaceMissingValueWith': 'missing_value',
|
|
'retainMissingValue': False,
|
|
'lookup': {
|
|
'type': 'map',
|
|
'map': {
|
|
'iPhone10,1': 'iPhone 8',
|
|
'iPhone10,4': 'iPhone 8',
|
|
'iPhone10,2': 'iPhone 8 Plus',
|
|
'iPhone10,5': 'iPhone 8 Plus',
|
|
'iPhone10,3': 'iPhone X',
|
|
'iPhone10,6': 'iPhone X',
|
|
},
|
|
'isOneToOne': False,
|
|
},
|
|
},
|
|
}
|
|
spec_json = json.dumps(dimension_spec)
|
|
col = DruidColumn(column_name='deviceName', dimension_spec_json=spec_json)
|
|
column_dict = {'deviceName': col}
|
|
f = DruidDatasource.get_filters(filters, [], column_dict)
|
|
assert isinstance(f.extraction_function, MapLookupExtraction)
|
|
dim_ext_fn = dimension_spec['extractionFn']
|
|
f_ext_fn = f.extraction_function
|
|
self.assertEqual(dim_ext_fn['lookup']['map'], f_ext_fn._mapping)
|
|
self.assertEqual(dim_ext_fn['lookup']['isOneToOne'], f_ext_fn._injective)
|
|
self.assertEqual(
|
|
dim_ext_fn['replaceMissingValueWith'],
|
|
f_ext_fn._replace_missing_values,
|
|
)
|
|
self.assertEqual(
|
|
dim_ext_fn['retainMissingValue'],
|
|
f_ext_fn._retain_missing_values,
|
|
)
|
|
|
|
def test_get_filters_extraction_fn_regex(self):
|
|
filters = [{'col': 'buildPrefix', 'val': ['22B'], 'op': 'in'}]
|
|
dimension_spec = {
|
|
'type': 'extraction',
|
|
'dimension': 'build',
|
|
'outputName': 'buildPrefix',
|
|
'outputType': 'STRING',
|
|
'extractionFn': {
|
|
'type': 'regex',
|
|
'expr': '(^[0-9A-Za-z]{3})',
|
|
},
|
|
}
|
|
spec_json = json.dumps(dimension_spec)
|
|
col = DruidColumn(column_name='buildPrefix', dimension_spec_json=spec_json)
|
|
column_dict = {'buildPrefix': col}
|
|
f = DruidDatasource.get_filters(filters, [], column_dict)
|
|
assert isinstance(f.extraction_function, RegexExtraction)
|
|
dim_ext_fn = dimension_spec['extractionFn']
|
|
f_ext_fn = f.extraction_function
|
|
self.assertEqual(dim_ext_fn['expr'], f_ext_fn._expr)
|
|
|
|
def test_get_filters_ignores_invalid_filter_objects(self):
|
|
filtr = {'col': 'col1', 'op': '=='}
|
|
filters = [filtr]
|
|
col = DruidColumn(column_name='col1')
|
|
column_dict = {'col1': col}
|
|
self.assertIsNone(DruidDatasource.get_filters(filters, [], column_dict))
|
|
|
|
def test_get_filters_constructs_filter_in(self):
|
|
filtr = {'col': 'A', 'op': 'in', 'val': ['a', 'b', 'c']}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertIn('filter', res.filter)
|
|
self.assertIn('fields', res.filter['filter'])
|
|
self.assertEqual('or', res.filter['filter']['type'])
|
|
self.assertEqual(3, len(res.filter['filter']['fields']))
|
|
|
|
def test_get_filters_constructs_filter_not_in(self):
|
|
filtr = {'col': 'A', 'op': 'not in', 'val': ['a', 'b', 'c']}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertIn('filter', res.filter)
|
|
self.assertIn('type', res.filter['filter'])
|
|
self.assertEqual('not', res.filter['filter']['type'])
|
|
self.assertIn('field', res.filter['filter'])
|
|
self.assertEqual(
|
|
3,
|
|
len(res.filter['filter']['field'].filter['filter']['fields']),
|
|
)
|
|
|
|
def test_get_filters_constructs_filter_equals(self):
|
|
filtr = {'col': 'A', 'op': '==', 'val': 'h'}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('selector', res.filter['filter']['type'])
|
|
self.assertEqual('A', res.filter['filter']['dimension'])
|
|
self.assertEqual('h', res.filter['filter']['value'])
|
|
|
|
def test_get_filters_constructs_filter_not_equals(self):
|
|
filtr = {'col': 'A', 'op': '!=', 'val': 'h'}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('not', res.filter['filter']['type'])
|
|
self.assertEqual(
|
|
'h',
|
|
res.filter['filter']['field'].filter['filter']['value'],
|
|
)
|
|
|
|
def test_get_filters_constructs_bounds_filter(self):
|
|
filtr = {'col': 'A', 'op': '>=', 'val': 'h'}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertFalse(res.filter['filter']['lowerStrict'])
|
|
self.assertEqual('A', res.filter['filter']['dimension'])
|
|
self.assertEqual('h', res.filter['filter']['lower'])
|
|
self.assertFalse(res.filter['filter']['alphaNumeric'])
|
|
filtr['op'] = '>'
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertTrue(res.filter['filter']['lowerStrict'])
|
|
filtr['op'] = '<='
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertFalse(res.filter['filter']['upperStrict'])
|
|
self.assertEqual('h', res.filter['filter']['upper'])
|
|
filtr['op'] = '<'
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertTrue(res.filter['filter']['upperStrict'])
|
|
|
|
def test_get_filters_constructs_regex_filter(self):
|
|
filtr = {'col': 'A', 'op': 'regex', 'val': '[abc]'}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('regex', res.filter['filter']['type'])
|
|
self.assertEqual('[abc]', res.filter['filter']['pattern'])
|
|
self.assertEqual('A', res.filter['filter']['dimension'])
|
|
|
|
def test_get_filters_composes_multiple_filters(self):
|
|
filtr1 = {'col': 'A', 'op': '!=', 'val': 'y'}
|
|
filtr2 = {'col': 'B', 'op': 'in', 'val': ['a', 'b', 'c']}
|
|
cola = DruidColumn(column_name='A')
|
|
colb = DruidColumn(column_name='B')
|
|
column_dict = {'A': cola, 'B': colb}
|
|
res = DruidDatasource.get_filters([filtr1, filtr2], [], column_dict)
|
|
self.assertEqual('and', res.filter['filter']['type'])
|
|
self.assertEqual(2, len(res.filter['filter']['fields']))
|
|
|
|
def test_get_filters_ignores_in_not_in_with_empty_value(self):
|
|
filtr1 = {'col': 'A', 'op': 'in', 'val': []}
|
|
filtr2 = {'col': 'A', 'op': 'not in', 'val': []}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr1, filtr2], [], column_dict)
|
|
self.assertIsNone(res)
|
|
|
|
def test_get_filters_constructs_equals_for_in_not_in_single_value(self):
|
|
filtr = {'col': 'A', 'op': 'in', 'val': ['a']}
|
|
cola = DruidColumn(column_name='A')
|
|
colb = DruidColumn(column_name='B')
|
|
column_dict = {'A': cola, 'B': colb}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('selector', res.filter['filter']['type'])
|
|
|
|
def test_get_filters_handles_arrays_for_string_types(self):
|
|
filtr = {'col': 'A', 'op': '==', 'val': ['a', 'b']}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('a', res.filter['filter']['value'])
|
|
|
|
filtr = {'col': 'A', 'op': '==', 'val': []}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertIsNone(res.filter['filter']['value'])
|
|
|
|
def test_get_filters_handles_none_for_string_types(self):
|
|
filtr = {'col': 'A', 'op': '==', 'val': None}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertIsNone(res)
|
|
|
|
def test_get_filters_extracts_values_in_quotes(self):
|
|
filtr = {'col': 'A', 'op': 'in', 'val': ['"a"']}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('a', res.filter['filter']['value'])
|
|
|
|
def test_get_filters_keeps_trailing_spaces(self):
|
|
filtr = {'col': 'A', 'op': 'in', 'val': ['a ']}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], [], column_dict)
|
|
self.assertEqual('a ', res.filter['filter']['value'])
|
|
|
|
def test_get_filters_converts_strings_to_num(self):
|
|
filtr = {'col': 'A', 'op': 'in', 'val': ['6']}
|
|
col = DruidColumn(column_name='A')
|
|
column_dict = {'A': col}
|
|
res = DruidDatasource.get_filters([filtr], ['A'], column_dict)
|
|
self.assertEqual(6, res.filter['filter']['value'])
|
|
filtr = {'col': 'A', 'op': '==', 'val': '6'}
|
|
res = DruidDatasource.get_filters([filtr], ['A'], column_dict)
|
|
self.assertEqual(6, res.filter['filter']['value'])
|
|
|
|
def test_run_query_no_groupby(self):
|
|
client = Mock()
|
|
from_dttm = Mock()
|
|
to_dttm = Mock()
|
|
from_dttm.replace = Mock(return_value=from_dttm)
|
|
to_dttm.replace = Mock(return_value=to_dttm)
|
|
from_dttm.isoformat = Mock(return_value='from')
|
|
to_dttm.isoformat = Mock(return_value='to')
|
|
timezone = 'timezone'
|
|
from_dttm.tzname = Mock(return_value=timezone)
|
|
ds = DruidDatasource(datasource_name='datasource')
|
|
metric1 = DruidMetric(metric_name='metric1')
|
|
metric2 = DruidMetric(metric_name='metric2')
|
|
ds.metrics = [metric1, metric2]
|
|
col1 = DruidColumn(column_name='col1')
|
|
col2 = DruidColumn(column_name='col2')
|
|
ds.columns = [col1, col2]
|
|
aggs = []
|
|
post_aggs = ['some_agg']
|
|
ds._metrics_and_post_aggs = Mock(return_value=(aggs, post_aggs))
|
|
groupby = []
|
|
metrics = ['metric1']
|
|
ds.get_having_filters = Mock(return_value=[])
|
|
client.query_builder = Mock()
|
|
client.query_builder.last_query = Mock()
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
# no groupby calls client.timeseries
|
|
ds.run_query(
|
|
groupby, metrics, None, from_dttm,
|
|
to_dttm, client=client, filter=[], row_limit=100,
|
|
)
|
|
self.assertEqual(0, len(client.topn.call_args_list))
|
|
self.assertEqual(0, len(client.groupby.call_args_list))
|
|
self.assertEqual(1, len(client.timeseries.call_args_list))
|
|
# check that there is no dimensions entry
|
|
called_args = client.timeseries.call_args_list[0][1]
|
|
self.assertNotIn('dimensions', called_args)
|
|
self.assertIn('post_aggregations', called_args)
|
|
# restore functions
|
|
|
|
def test_run_query_with_adhoc_metric(self):
|
|
client = Mock()
|
|
from_dttm = Mock()
|
|
to_dttm = Mock()
|
|
from_dttm.replace = Mock(return_value=from_dttm)
|
|
to_dttm.replace = Mock(return_value=to_dttm)
|
|
from_dttm.isoformat = Mock(return_value='from')
|
|
to_dttm.isoformat = Mock(return_value='to')
|
|
timezone = 'timezone'
|
|
from_dttm.tzname = Mock(return_value=timezone)
|
|
ds = DruidDatasource(datasource_name='datasource')
|
|
metric1 = DruidMetric(metric_name='metric1')
|
|
metric2 = DruidMetric(metric_name='metric2')
|
|
ds.metrics = [metric1, metric2]
|
|
col1 = DruidColumn(column_name='col1')
|
|
col2 = DruidColumn(column_name='col2')
|
|
ds.columns = [col1, col2]
|
|
all_metrics = []
|
|
post_aggs = ['some_agg']
|
|
ds._metrics_and_post_aggs = Mock(return_value=(all_metrics, post_aggs))
|
|
groupby = []
|
|
metrics = [{
|
|
'expressionType': 'SIMPLE',
|
|
'column': {'type': 'DOUBLE', 'column_name': 'col1'},
|
|
'aggregate': 'SUM',
|
|
'label': 'My Adhoc Metric',
|
|
}]
|
|
|
|
ds.get_having_filters = Mock(return_value=[])
|
|
client.query_builder = Mock()
|
|
client.query_builder.last_query = Mock()
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
# no groupby calls client.timeseries
|
|
ds.run_query(
|
|
groupby, metrics, None, from_dttm,
|
|
to_dttm, client=client, filter=[], row_limit=100,
|
|
)
|
|
self.assertEqual(0, len(client.topn.call_args_list))
|
|
self.assertEqual(0, len(client.groupby.call_args_list))
|
|
self.assertEqual(1, len(client.timeseries.call_args_list))
|
|
# check that there is no dimensions entry
|
|
called_args = client.timeseries.call_args_list[0][1]
|
|
self.assertNotIn('dimensions', called_args)
|
|
self.assertIn('post_aggregations', called_args)
|
|
# restore functions
|
|
|
|
def test_run_query_single_groupby(self):
|
|
client = Mock()
|
|
from_dttm = Mock()
|
|
to_dttm = Mock()
|
|
from_dttm.replace = Mock(return_value=from_dttm)
|
|
to_dttm.replace = Mock(return_value=to_dttm)
|
|
from_dttm.isoformat = Mock(return_value='from')
|
|
to_dttm.isoformat = Mock(return_value='to')
|
|
timezone = 'timezone'
|
|
from_dttm.tzname = Mock(return_value=timezone)
|
|
ds = DruidDatasource(datasource_name='datasource')
|
|
metric1 = DruidMetric(metric_name='metric1')
|
|
metric2 = DruidMetric(metric_name='metric2')
|
|
ds.metrics = [metric1, metric2]
|
|
col1 = DruidColumn(column_name='col1')
|
|
col2 = DruidColumn(column_name='col2')
|
|
ds.columns = [col1, col2]
|
|
aggs = ['metric1']
|
|
post_aggs = ['some_agg']
|
|
ds._metrics_and_post_aggs = Mock(return_value=(aggs, post_aggs))
|
|
groupby = ['col1']
|
|
metrics = ['metric1']
|
|
ds.get_having_filters = Mock(return_value=[])
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
# client.topn is called twice
|
|
ds.run_query(
|
|
groupby, metrics, None, from_dttm, to_dttm, timeseries_limit=100,
|
|
client=client, order_desc=True, filter=[],
|
|
)
|
|
self.assertEqual(2, len(client.topn.call_args_list))
|
|
self.assertEqual(0, len(client.groupby.call_args_list))
|
|
self.assertEqual(0, len(client.timeseries.call_args_list))
|
|
# check that there is no dimensions entry
|
|
called_args_pre = client.topn.call_args_list[0][1]
|
|
self.assertNotIn('dimensions', called_args_pre)
|
|
self.assertIn('dimension', called_args_pre)
|
|
called_args = client.topn.call_args_list[1][1]
|
|
self.assertIn('dimension', called_args)
|
|
self.assertEqual('col1', called_args['dimension'])
|
|
# not order_desc
|
|
client = Mock()
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
ds.run_query(
|
|
groupby, metrics, None, from_dttm, to_dttm, client=client,
|
|
order_desc=False, filter=[], row_limit=100,
|
|
)
|
|
self.assertEqual(0, len(client.topn.call_args_list))
|
|
self.assertEqual(1, len(client.groupby.call_args_list))
|
|
self.assertEqual(0, len(client.timeseries.call_args_list))
|
|
self.assertIn('dimensions', client.groupby.call_args_list[0][1])
|
|
self.assertEqual(['col1'], client.groupby.call_args_list[0][1]['dimensions'])
|
|
# order_desc but timeseries and dimension spec
|
|
# calls topn with single dimension spec 'dimension'
|
|
spec = {'outputName': 'hello', 'dimension': 'matcho'}
|
|
spec_json = json.dumps(spec)
|
|
col3 = DruidColumn(column_name='col3', dimension_spec_json=spec_json)
|
|
ds.columns.append(col3)
|
|
groupby = ['col3']
|
|
client = Mock()
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
ds.run_query(
|
|
groupby, metrics, None, from_dttm, to_dttm,
|
|
client=client, order_desc=True, timeseries_limit=5,
|
|
filter=[], row_limit=100,
|
|
)
|
|
self.assertEqual(2, len(client.topn.call_args_list))
|
|
self.assertEqual(0, len(client.groupby.call_args_list))
|
|
self.assertEqual(0, len(client.timeseries.call_args_list))
|
|
self.assertIn('dimension', client.topn.call_args_list[0][1])
|
|
self.assertIn('dimension', client.topn.call_args_list[1][1])
|
|
# uses dimension for pre query and full spec for final query
|
|
self.assertEqual('matcho', client.topn.call_args_list[0][1]['dimension'])
|
|
self.assertEqual(spec, client.topn.call_args_list[1][1]['dimension'])
|
|
|
|
def test_run_query_multiple_groupby(self):
|
|
client = Mock()
|
|
from_dttm = Mock()
|
|
to_dttm = Mock()
|
|
from_dttm.replace = Mock(return_value=from_dttm)
|
|
to_dttm.replace = Mock(return_value=to_dttm)
|
|
from_dttm.isoformat = Mock(return_value='from')
|
|
to_dttm.isoformat = Mock(return_value='to')
|
|
timezone = 'timezone'
|
|
from_dttm.tzname = Mock(return_value=timezone)
|
|
ds = DruidDatasource(datasource_name='datasource')
|
|
metric1 = DruidMetric(metric_name='metric1')
|
|
metric2 = DruidMetric(metric_name='metric2')
|
|
ds.metrics = [metric1, metric2]
|
|
col1 = DruidColumn(column_name='col1')
|
|
col2 = DruidColumn(column_name='col2')
|
|
ds.columns = [col1, col2]
|
|
aggs = []
|
|
post_aggs = ['some_agg']
|
|
ds._metrics_and_post_aggs = Mock(return_value=(aggs, post_aggs))
|
|
groupby = ['col1', 'col2']
|
|
metrics = ['metric1']
|
|
ds.get_having_filters = Mock(return_value=[])
|
|
client.query_builder = Mock()
|
|
client.query_builder.last_query = Mock()
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
# no groupby calls client.timeseries
|
|
ds.run_query(
|
|
groupby, metrics, None, from_dttm,
|
|
to_dttm, client=client, row_limit=100,
|
|
filter=[],
|
|
)
|
|
self.assertEqual(0, len(client.topn.call_args_list))
|
|
self.assertEqual(1, len(client.groupby.call_args_list))
|
|
self.assertEqual(0, len(client.timeseries.call_args_list))
|
|
# check that there is no dimensions entry
|
|
called_args = client.groupby.call_args_list[0][1]
|
|
self.assertIn('dimensions', called_args)
|
|
self.assertEqual(['col1', 'col2'], called_args['dimensions'])
|
|
|
|
def test_get_post_agg_returns_correct_agg_type(self):
|
|
get_post_agg = DruidDatasource.get_post_agg
|
|
# javascript PostAggregators
|
|
function = 'function(field1, field2) { return field1 + field2; }'
|
|
conf = {
|
|
'type': 'javascript',
|
|
'name': 'postagg_name',
|
|
'fieldNames': ['field1', 'field2'],
|
|
'function': function,
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, models.JavascriptPostAggregator))
|
|
self.assertEqual(postagg.name, 'postagg_name')
|
|
self.assertEqual(postagg.post_aggregator['type'], 'javascript')
|
|
self.assertEqual(postagg.post_aggregator['fieldNames'], ['field1', 'field2'])
|
|
self.assertEqual(postagg.post_aggregator['name'], 'postagg_name')
|
|
self.assertEqual(postagg.post_aggregator['function'], function)
|
|
# Quantile
|
|
conf = {
|
|
'type': 'quantile',
|
|
'name': 'postagg_name',
|
|
'probability': '0.5',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, postaggs.Quantile))
|
|
self.assertEqual(postagg.name, 'postagg_name')
|
|
self.assertEqual(postagg.post_aggregator['probability'], '0.5')
|
|
# Quantiles
|
|
conf = {
|
|
'type': 'quantiles',
|
|
'name': 'postagg_name',
|
|
'probabilities': '0.4,0.5,0.6',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, postaggs.Quantiles))
|
|
self.assertEqual(postagg.name, 'postagg_name')
|
|
self.assertEqual(postagg.post_aggregator['probabilities'], '0.4,0.5,0.6')
|
|
# FieldAccess
|
|
conf = {
|
|
'type': 'fieldAccess',
|
|
'name': 'field_name',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, postaggs.Field))
|
|
self.assertEqual(postagg.name, 'field_name')
|
|
# constant
|
|
conf = {
|
|
'type': 'constant',
|
|
'value': 1234,
|
|
'name': 'postagg_name',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, postaggs.Const))
|
|
self.assertEqual(postagg.name, 'postagg_name')
|
|
self.assertEqual(postagg.post_aggregator['value'], 1234)
|
|
# hyperUniqueCardinality
|
|
conf = {
|
|
'type': 'hyperUniqueCardinality',
|
|
'name': 'unique_name',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, postaggs.HyperUniqueCardinality))
|
|
self.assertEqual(postagg.name, 'unique_name')
|
|
# arithmetic
|
|
conf = {
|
|
'type': 'arithmetic',
|
|
'fn': '+',
|
|
'fields': ['field1', 'field2'],
|
|
'name': 'postagg_name',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, postaggs.Postaggregator))
|
|
self.assertEqual(postagg.name, 'postagg_name')
|
|
self.assertEqual(postagg.post_aggregator['fn'], '+')
|
|
self.assertEqual(postagg.post_aggregator['fields'], ['field1', 'field2'])
|
|
# custom post aggregator
|
|
conf = {
|
|
'type': 'custom',
|
|
'name': 'custom_name',
|
|
'stuff': 'more_stuff',
|
|
}
|
|
postagg = get_post_agg(conf)
|
|
self.assertTrue(isinstance(postagg, models.CustomPostAggregator))
|
|
self.assertEqual(postagg.name, 'custom_name')
|
|
self.assertEqual(postagg.post_aggregator['stuff'], 'more_stuff')
|
|
|
|
def test_find_postaggs_for_returns_postaggs_and_removes(self):
|
|
find_postaggs_for = DruidDatasource.find_postaggs_for
|
|
postagg_names = set(['pa2', 'pa3', 'pa4', 'm1', 'm2', 'm3', 'm4'])
|
|
|
|
metrics = {}
|
|
for i in range(1, 6):
|
|
emplace(metrics, 'pa' + str(i), True)
|
|
emplace(metrics, 'm' + str(i), False)
|
|
postagg_list = find_postaggs_for(postagg_names, metrics)
|
|
self.assertEqual(3, len(postagg_list))
|
|
self.assertEqual(4, len(postagg_names))
|
|
expected_metrics = ['m1', 'm2', 'm3', 'm4']
|
|
expected_postaggs = set(['pa2', 'pa3', 'pa4'])
|
|
for postagg in postagg_list:
|
|
expected_postaggs.remove(postagg.metric_name)
|
|
for metric in expected_metrics:
|
|
postagg_names.remove(metric)
|
|
self.assertEqual(0, len(expected_postaggs))
|
|
self.assertEqual(0, len(postagg_names))
|
|
|
|
def test_recursive_get_fields(self):
|
|
conf = {
|
|
'type': 'quantile',
|
|
'fieldName': 'f1',
|
|
'field': {
|
|
'type': 'custom',
|
|
'fields': [{
|
|
'type': 'fieldAccess',
|
|
'fieldName': 'f2',
|
|
}, {
|
|
'type': 'fieldAccess',
|
|
'fieldName': 'f3',
|
|
}, {
|
|
'type': 'quantiles',
|
|
'fieldName': 'f4',
|
|
'field': {
|
|
'type': 'custom',
|
|
},
|
|
}, {
|
|
'type': 'custom',
|
|
'fields': [{
|
|
'type': 'fieldAccess',
|
|
'fieldName': 'f5',
|
|
}, {
|
|
'type': 'fieldAccess',
|
|
'fieldName': 'f2',
|
|
'fields': [{
|
|
'type': 'fieldAccess',
|
|
'fieldName': 'f3',
|
|
}, {
|
|
'type': 'fieldIgnoreMe',
|
|
'fieldName': 'f6',
|
|
}],
|
|
}],
|
|
}],
|
|
},
|
|
}
|
|
fields = DruidDatasource.recursive_get_fields(conf)
|
|
expected = set(['f1', 'f2', 'f3', 'f4', 'f5'])
|
|
self.assertEqual(5, len(fields))
|
|
for field in fields:
|
|
expected.remove(field)
|
|
self.assertEqual(0, len(expected))
|
|
|
|
def test_metrics_and_post_aggs_tree(self):
|
|
metrics = ['A', 'B', 'm1', 'm2']
|
|
metrics_dict = {}
|
|
for i in range(ord('A'), ord('K') + 1):
|
|
emplace(metrics_dict, chr(i), True)
|
|
for i in range(1, 10):
|
|
emplace(metrics_dict, 'm' + str(i), False)
|
|
|
|
def depends_on(index, fields):
|
|
dependents = fields if isinstance(fields, list) else [fields]
|
|
metrics_dict[index].json_obj = {'fieldNames': dependents}
|
|
|
|
depends_on('A', ['m1', 'D', 'C'])
|
|
depends_on('B', ['B', 'C', 'E', 'F', 'm3'])
|
|
depends_on('C', ['H', 'I'])
|
|
depends_on('D', ['m2', 'm5', 'G', 'C'])
|
|
depends_on('E', ['H', 'I', 'J'])
|
|
depends_on('F', ['J', 'm5'])
|
|
depends_on('G', ['m4', 'm7', 'm6', 'A'])
|
|
depends_on('H', ['A', 'm4', 'I'])
|
|
depends_on('I', ['H', 'K'])
|
|
depends_on('J', 'K')
|
|
depends_on('K', ['m8', 'm9'])
|
|
aggs, postaggs = DruidDatasource.metrics_and_post_aggs(
|
|
metrics, metrics_dict)
|
|
expected_metrics = set(aggs.keys())
|
|
self.assertEqual(9, len(aggs))
|
|
for i in range(1, 10):
|
|
expected_metrics.remove('m' + str(i))
|
|
self.assertEqual(0, len(expected_metrics))
|
|
self.assertEqual(11, len(postaggs))
|
|
for i in range(ord('A'), ord('K') + 1):
|
|
del postaggs[chr(i)]
|
|
self.assertEqual(0, len(postaggs))
|
|
|
|
def test_metrics_and_post_aggs(self):
|
|
"""
|
|
Test generation of metrics and post-aggregations from an initial list
|
|
of superset metrics (which may include the results of either). This
|
|
primarily tests that specifying a post-aggregator metric will also
|
|
require the raw aggregation of the associated druid metric column.
|
|
"""
|
|
metrics_dict = {
|
|
'unused_count': DruidMetric(
|
|
metric_name='unused_count',
|
|
verbose_name='COUNT(*)',
|
|
metric_type='count',
|
|
json=json.dumps({'type': 'count', 'name': 'unused_count'}),
|
|
),
|
|
'some_sum': DruidMetric(
|
|
metric_name='some_sum',
|
|
verbose_name='SUM(*)',
|
|
metric_type='sum',
|
|
json=json.dumps({'type': 'sum', 'name': 'sum'}),
|
|
),
|
|
'a_histogram': DruidMetric(
|
|
metric_name='a_histogram',
|
|
verbose_name='APPROXIMATE_HISTOGRAM(*)',
|
|
metric_type='approxHistogramFold',
|
|
json=json.dumps(
|
|
{'type': 'approxHistogramFold', 'name': 'a_histogram'},
|
|
),
|
|
),
|
|
'aCustomMetric': DruidMetric(
|
|
metric_name='aCustomMetric',
|
|
verbose_name='MY_AWESOME_METRIC(*)',
|
|
metric_type='aCustomType',
|
|
json=json.dumps(
|
|
{'type': 'customMetric', 'name': 'aCustomMetric'},
|
|
),
|
|
),
|
|
'quantile_p95': DruidMetric(
|
|
metric_name='quantile_p95',
|
|
verbose_name='P95(*)',
|
|
metric_type='postagg',
|
|
json=json.dumps({
|
|
'type': 'quantile',
|
|
'probability': 0.95,
|
|
'name': 'p95',
|
|
'fieldName': 'a_histogram',
|
|
}),
|
|
),
|
|
'aCustomPostAgg': DruidMetric(
|
|
metric_name='aCustomPostAgg',
|
|
verbose_name='CUSTOM_POST_AGG(*)',
|
|
metric_type='postagg',
|
|
json=json.dumps({
|
|
'type': 'customPostAgg',
|
|
'name': 'aCustomPostAgg',
|
|
'field': {
|
|
'type': 'fieldAccess',
|
|
'fieldName': 'aCustomMetric',
|
|
},
|
|
}),
|
|
),
|
|
}
|
|
|
|
adhoc_metric = {
|
|
'expressionType': 'SIMPLE',
|
|
'column': {'type': 'DOUBLE', 'column_name': 'value'},
|
|
'aggregate': 'SUM',
|
|
'label': 'My Adhoc Metric',
|
|
}
|
|
|
|
metrics = ['some_sum']
|
|
saved_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
|
|
metrics, metrics_dict)
|
|
|
|
assert set(saved_metrics.keys()) == {'some_sum'}
|
|
assert post_aggs == {}
|
|
|
|
metrics = [adhoc_metric]
|
|
saved_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
|
|
metrics, metrics_dict)
|
|
|
|
assert set(saved_metrics.keys()) == set([adhoc_metric['label']])
|
|
assert post_aggs == {}
|
|
|
|
metrics = ['some_sum', adhoc_metric]
|
|
saved_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
|
|
metrics, metrics_dict)
|
|
|
|
assert set(saved_metrics.keys()) == {'some_sum', adhoc_metric['label']}
|
|
assert post_aggs == {}
|
|
|
|
metrics = ['quantile_p95']
|
|
saved_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
|
|
metrics, metrics_dict)
|
|
|
|
result_postaggs = set(['quantile_p95'])
|
|
assert set(saved_metrics.keys()) == {'a_histogram'}
|
|
assert set(post_aggs.keys()) == result_postaggs
|
|
|
|
metrics = ['aCustomPostAgg']
|
|
saved_metrics, post_aggs = DruidDatasource.metrics_and_post_aggs(
|
|
metrics, metrics_dict)
|
|
|
|
result_postaggs = set(['aCustomPostAgg'])
|
|
assert set(saved_metrics.keys()) == {'aCustomMetric'}
|
|
assert set(post_aggs.keys()) == result_postaggs
|
|
|
|
def test_druid_type_from_adhoc_metric(self):
|
|
|
|
druid_type = DruidDatasource.druid_type_from_adhoc_metric({
|
|
'column': {'type': 'DOUBLE', 'column_name': 'value'},
|
|
'aggregate': 'SUM',
|
|
'label': 'My Adhoc Metric',
|
|
})
|
|
assert(druid_type == 'doubleSum')
|
|
|
|
druid_type = DruidDatasource.druid_type_from_adhoc_metric({
|
|
'column': {'type': 'LONG', 'column_name': 'value'},
|
|
'aggregate': 'MAX',
|
|
'label': 'My Adhoc Metric',
|
|
})
|
|
assert(druid_type == 'longMax')
|
|
|
|
druid_type = DruidDatasource.druid_type_from_adhoc_metric({
|
|
'column': {'type': 'VARCHAR(255)', 'column_name': 'value'},
|
|
'aggregate': 'COUNT',
|
|
'label': 'My Adhoc Metric',
|
|
})
|
|
assert(druid_type == 'count')
|
|
|
|
druid_type = DruidDatasource.druid_type_from_adhoc_metric({
|
|
'column': {'type': 'VARCHAR(255)', 'column_name': 'value'},
|
|
'aggregate': 'COUNT_DISTINCT',
|
|
'label': 'My Adhoc Metric',
|
|
})
|
|
assert(druid_type == 'cardinality')
|
|
|
|
druid_type = DruidDatasource.druid_type_from_adhoc_metric({
|
|
'column': {'type': 'hyperUnique', 'column_name': 'value'},
|
|
'aggregate': 'COUNT_DISTINCT',
|
|
'label': 'My Adhoc Metric',
|
|
})
|
|
assert(druid_type == 'hyperUnique')
|
|
|
|
def test_run_query_order_by_metrics(self):
|
|
client = Mock()
|
|
client.query_builder.last_query.query_dict = {'mock': 0}
|
|
from_dttm = Mock()
|
|
to_dttm = Mock()
|
|
ds = DruidDatasource(datasource_name='datasource')
|
|
ds.get_having_filters = Mock(return_value=[])
|
|
dim1 = DruidColumn(column_name='dim1')
|
|
dim2 = DruidColumn(column_name='dim2')
|
|
metrics_dict = {
|
|
'count1': DruidMetric(
|
|
metric_name='count1',
|
|
metric_type='count',
|
|
json=json.dumps({'type': 'count', 'name': 'count1'}),
|
|
),
|
|
'sum1': DruidMetric(
|
|
metric_name='sum1',
|
|
metric_type='doubleSum',
|
|
json=json.dumps({'type': 'doubleSum', 'name': 'sum1'}),
|
|
),
|
|
'sum2': DruidMetric(
|
|
metric_name='sum2',
|
|
metric_type='doubleSum',
|
|
json=json.dumps({'type': 'doubleSum', 'name': 'sum2'}),
|
|
),
|
|
'div1': DruidMetric(
|
|
metric_name='div1',
|
|
metric_type='postagg',
|
|
json=json.dumps({
|
|
'fn': '/',
|
|
'type': 'arithmetic',
|
|
'name': 'div1',
|
|
'fields': [
|
|
{
|
|
'fieldName': 'sum1',
|
|
'type': 'fieldAccess',
|
|
},
|
|
{
|
|
'fieldName': 'sum2',
|
|
'type': 'fieldAccess',
|
|
},
|
|
],
|
|
}),
|
|
),
|
|
}
|
|
ds.columns = [dim1, dim2]
|
|
ds.metrics = list(metrics_dict.values())
|
|
|
|
groupby = ['dim1']
|
|
metrics = ['count1']
|
|
granularity = 'all'
|
|
# get the counts of the top 5 'dim1's, order by 'sum1'
|
|
ds.run_query(
|
|
groupby, metrics, granularity, from_dttm, to_dttm,
|
|
timeseries_limit=5, timeseries_limit_metric='sum1',
|
|
client=client, order_desc=True, filter=[],
|
|
)
|
|
qry_obj = client.topn.call_args_list[0][1]
|
|
self.assertEqual('dim1', qry_obj['dimension'])
|
|
self.assertEqual('sum1', qry_obj['metric'])
|
|
aggregations = qry_obj['aggregations']
|
|
post_aggregations = qry_obj['post_aggregations']
|
|
self.assertEqual({'count1', 'sum1'}, set(aggregations.keys()))
|
|
self.assertEqual(set(), set(post_aggregations.keys()))
|
|
|
|
# get the counts of the top 5 'dim1's, order by 'div1'
|
|
ds.run_query(
|
|
groupby, metrics, granularity, from_dttm, to_dttm,
|
|
timeseries_limit=5, timeseries_limit_metric='div1',
|
|
client=client, order_desc=True, filter=[],
|
|
)
|
|
qry_obj = client.topn.call_args_list[1][1]
|
|
self.assertEqual('dim1', qry_obj['dimension'])
|
|
self.assertEqual('div1', qry_obj['metric'])
|
|
aggregations = qry_obj['aggregations']
|
|
post_aggregations = qry_obj['post_aggregations']
|
|
self.assertEqual({'count1', 'sum1', 'sum2'}, set(aggregations.keys()))
|
|
self.assertEqual({'div1'}, set(post_aggregations.keys()))
|
|
|
|
groupby = ['dim1', 'dim2']
|
|
# get the counts of the top 5 ['dim1', 'dim2']s, order by 'sum1'
|
|
ds.run_query(
|
|
groupby, metrics, granularity, from_dttm, to_dttm,
|
|
timeseries_limit=5, timeseries_limit_metric='sum1',
|
|
client=client, order_desc=True, filter=[],
|
|
)
|
|
qry_obj = client.groupby.call_args_list[0][1]
|
|
self.assertEqual({'dim1', 'dim2'}, set(qry_obj['dimensions']))
|
|
self.assertEqual('sum1', qry_obj['limit_spec']['columns'][0]['dimension'])
|
|
aggregations = qry_obj['aggregations']
|
|
post_aggregations = qry_obj['post_aggregations']
|
|
self.assertEqual({'count1', 'sum1'}, set(aggregations.keys()))
|
|
self.assertEqual(set(), set(post_aggregations.keys()))
|
|
|
|
# get the counts of the top 5 ['dim1', 'dim2']s, order by 'div1'
|
|
ds.run_query(
|
|
groupby, metrics, granularity, from_dttm, to_dttm,
|
|
timeseries_limit=5, timeseries_limit_metric='div1',
|
|
client=client, order_desc=True, filter=[],
|
|
)
|
|
qry_obj = client.groupby.call_args_list[1][1]
|
|
self.assertEqual({'dim1', 'dim2'}, set(qry_obj['dimensions']))
|
|
self.assertEqual('div1', qry_obj['limit_spec']['columns'][0]['dimension'])
|
|
aggregations = qry_obj['aggregations']
|
|
post_aggregations = qry_obj['post_aggregations']
|
|
self.assertEqual({'count1', 'sum1', 'sum2'}, set(aggregations.keys()))
|
|
self.assertEqual({'div1'}, set(post_aggregations.keys()))
|
|
|
|
def test_get_aggregations(self):
|
|
ds = DruidDatasource(datasource_name='datasource')
|
|
metrics_dict = {
|
|
'sum1': DruidMetric(
|
|
metric_name='sum1',
|
|
metric_type='doubleSum',
|
|
json=json.dumps({'type': 'doubleSum', 'name': 'sum1'}),
|
|
),
|
|
'sum2': DruidMetric(
|
|
metric_name='sum2',
|
|
metric_type='doubleSum',
|
|
json=json.dumps({'type': 'doubleSum', 'name': 'sum2'}),
|
|
),
|
|
'div1': DruidMetric(
|
|
metric_name='div1',
|
|
metric_type='postagg',
|
|
json=json.dumps({
|
|
'fn': '/',
|
|
'type': 'arithmetic',
|
|
'name': 'div1',
|
|
'fields': [
|
|
{
|
|
'fieldName': 'sum1',
|
|
'type': 'fieldAccess',
|
|
},
|
|
{
|
|
'fieldName': 'sum2',
|
|
'type': 'fieldAccess',
|
|
},
|
|
],
|
|
}),
|
|
),
|
|
}
|
|
metric_names = ['sum1', 'sum2']
|
|
aggs = ds.get_aggregations(metrics_dict, metric_names)
|
|
expected_agg = {name: metrics_dict[name].json_obj for name in metric_names}
|
|
self.assertEqual(expected_agg, aggs)
|
|
|
|
metric_names = ['sum1', 'col1']
|
|
self.assertRaises(
|
|
SupersetException, ds.get_aggregations, metrics_dict, metric_names)
|
|
|
|
metric_names = ['sum1', 'div1']
|
|
self.assertRaises(
|
|
SupersetException, ds.get_aggregations, metrics_dict, metric_names)
|