pt/superset
1
0
Fork 0
mirror of https://github.com/apache/superset.git synced 2024-09-19 12:09:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: implement extra filter logic (#688)

Browse Source 
* fix: implement extra filter logic

* fix bugs and add tests

* remove redundant changes

* improve types

* fix coverage

* improve codevov
This commit is contained in:
Ville Brofeldt 2020-07-21 06:24:34 +03:00 committed by Yongjie Zhao
parent a62559ea54
commit ea729831c3
9 changed files with 209 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/src/buildQueryObject.ts
View File

@ -1,18 +1,14 @@
/* eslint-disable camelcase */
import { QueryObject } from './types/Query';
import { isSqlaFormData, QueryFormData } from './types/QueryFormData';
import { QueryFormData } from './types/QueryFormData';
import processGroupby from './processGroupby';
import convertMetric from './convertMetric';
import processFilters from './processFilters';
import processExtras from './processExtras';
import extractExtras from './extractExtras';
import extractQueryFields from './extractQueryFields';
export const DTTM_ALIAS = '__timestamp';
function processGranularity(formData: QueryFormData): string {
return isSqlaFormData(formData) ? formData.granularity_sqla : formData.granularity;
}
/**
* Build the common segments of all query objects (e.g. the granularity field derived from
* either sql alchemy or druid). The segments specific to each viz type is constructed in the
@ -31,6 +27,7 @@ export default function buildQueryObject<T extends QueryFormData>(formData: T):
limit,
timeseries_limit_metric,
queryFields,
granularity,
...residualFormData
} = formData;
@ -39,9 +36,19 @@ export default function buildQueryObject<T extends QueryFormData>(formData: T):
const { metrics, groupby, columns } = extractQueryFields(residualFormData, queryFields);
const groupbySet = new Set([...columns, ...groupby]);
const extraFilters = extractExtras(formData);
const extrasAndfilters = processFilters({
...formData,
...extraFilters,
});
return {
extras: processExtras(formData),
granularity: processGranularity(formData),
time_range,
since,
until,
granularity,
...extraFilters,
...extrasAndfilters,
groupby: processGroupby(Array.from(groupbySet)),
is_timeseries: groupbySet.has(DTTM_ALIAS),
metrics: metrics.map(convertMetric),
@ -49,13 +56,9 @@ export default function buildQueryObject<T extends QueryFormData>(formData: T):
orderby: [],
row_limit: row_limit == null || Number.isNaN(numericRowLimit) ? undefined : numericRowLimit,
row_offset: row_offset == null || Number.isNaN(numericRowOffset) ? undefined : numericRowOffset,
since,
time_range,
timeseries_limit: limit ? Number(limit) : 0,
timeseries_limit_metric: timeseries_limit_metric
? convertMetric(timeseries_limit_metric)
: null,
until,
...processFilters(formData),
};
}

47
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/src/extractExtras.ts Normal file
View File

@ -0,0 +1,47 @@
/* eslint-disable camelcase */
import { isDruidFormData, QueryFormData } from './types/QueryFormData';
import { QueryObject } from './types/Query';
export default function extractExtras(formData: QueryFormData): Partial<QueryObject> {
const partialQueryObject: Partial<QueryObject> = {
filters: formData.filters || [],
extras: formData.extras || {},
};
const reservedColumnsToQueryField: Record<string, keyof QueryObject> = {
__time_range: 'time_range',
__time_col: 'granularity_sqla',
__time_grain: 'time_grain_sqla',
__time_origin: 'druid_time_origin',
__granularity: 'granularity',
};
(formData.extra_filters || []).forEach(filter => {
if (filter.col in reservedColumnsToQueryField) {
const queryField = reservedColumnsToQueryField[filter.col];
partialQueryObject[queryField] = filter.val;
} else {
// @ts-ignore
partialQueryObject.filters.push(filter);
}
});
// map to undeprecated names and remove deprecated fields
if (isDruidFormData(formData) && !partialQueryObject.druid_time_origin) {
partialQueryObject.extras = {
druid_time_origin: formData.druid_time_origin,
};
delete partialQueryObject.druid_time_origin;
} else {
// SQL
partialQueryObject.extras = {
...partialQueryObject.extras,
time_grain_sqla: partialQueryObject.time_grain_sqla || formData.time_grain_sqla,
};
partialQueryObject.granularity =
partialQueryObject.granularity_sqla || formData.granularity || formData.granularity_sqla;
delete partialQueryObject.granularity_sqla;
delete partialQueryObject.time_grain_sqla;
}
return partialQueryObject;
}

17
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/src/processExtras.ts
View File

@ -1,17 +0,0 @@
/* eslint-disable camelcase */
import { QueryFormData, isDruidFormData } from './types/QueryFormData';
import { QueryObjectExtras } from './types/Query';
export default function processExtras(formData: QueryFormData): QueryObjectExtras {
const { where = '' } = formData;
if (isDruidFormData(formData)) {
const { druid_time_origin, having_druid } = formData;
return { druid_time_origin, having_druid, where };
}
const { time_grain_sqla, having } = formData;
return { having, time_grain_sqla, where };
}

25
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/src/processFilters.ts
View File

@ -1,3 +1,4 @@
/* eslint-disable camelcase */
import { QueryFormData } from './types/QueryFormData';
import { QueryObjectFilterClause } from './types/Query';
import { isSimpleAdhocFilter } from './types/Filter';
@ -5,24 +6,17 @@ import convertFilter from './convertFilter';
/** Logic formerly in viz.py's process_query_filters */
export default function processFilters(formData: QueryFormData) {
// TODO: Implement
// utils.convert_legacy_filters_into_adhoc(self.form_data)
// TODO: Implement
// merge_extra_filters(self.form_data)
// Split adhoc_filters into four fields according to
// (1) clause (WHERE or HAVING)
// (2) expressionType
// 2.1 SIMPLE (subject + operator + comparator)
// 2.2 SQL (freeform SQL expression))
// eslint-disable-next-line camelcase
const { adhoc_filters } = formData;
if (Array.isArray(adhoc_filters)) {
const simpleWhere: QueryObjectFilterClause[] = [];
const simpleWhere: QueryObjectFilterClause[] = formData.filters || [];
const simpleHaving: QueryObjectFilterClause[] = [];
const freeformWhere: string[] = [];
if (formData.where) freeformWhere.push(formData.where);
const freeformHaving: string[] = [];
adhoc_filters.forEach(filter => {
@ -44,11 +38,18 @@ export default function processFilters(formData: QueryFormData) {
}
});
return {
filters: simpleWhere,
// some filter-related fields need to go in `extras`
const extras = {
having: freeformHaving.map(exp => `(${exp})`).join(' AND '),
having_filters: simpleHaving,
having_druid: simpleHaving,
where: freeformWhere.map(exp => `(${exp})`).join(' AND '),
...formData.extras,
};
return {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
filters: (formData.filters || []).concat(simpleWhere),
extras,
};
}

2
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/src/types/Query.ts
View File

@ -54,7 +54,7 @@ export type QueryObject = {
extras?: QueryObjectExtras;
/** Granularity (for steps in time series) */
granularity: string;
granularity?: string;
/** Free-form WHERE SQL: multiple clauses are concatenated by AND */
where?: string;

21
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/src/types/QueryFormData.ts
View File

@ -3,6 +3,7 @@
import { AdhocMetric } from './Metric';
import { TimeRange } from './Time';
import { AdhocFilter } from './Filter';
import { BinaryOperator, SetOperator } from './Operator';
export type QueryFormDataMetric = string | AdhocMetric;
export type QueryFormResidualDataValue = string | AdhocMetric;
@ -10,10 +11,25 @@ export type QueryFormResidualData = {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
[key: string]: any;
};
// Currently only Binary and Set filters are supported
export type QueryFields = {
[key: string]: string;
};
export type QueryFormExtraFilter = {
col: string;
} & (
| {
op: BinaryOperator;
val: string;
}
| {
op: SetOperator;
val: string[];
}
);
// Type signature for formData shared by all viz types
// It will be gradually filled out as we build out the query object
@ -37,6 +53,7 @@ export type BaseFormData = {
all_columns?: string[];
/** list of filters */
adhoc_filters?: AdhocFilter[];
extra_filters?: QueryFormExtraFilter[];
/** order descending */
order_desc?: boolean;
/** limit number of time series */
@ -77,7 +94,3 @@ export type QueryFormData = SqlaFormData | DruidFormData;
export function isDruidFormData(formData: QueryFormData): formData is DruidFormData {
return 'granularity' in formData;
}
export function isSqlaFormData(formData: QueryFormData): formData is SqlaFormData {
return 'granularity_sqla' in formData;
}

87
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/test/extractExtras.test.ts Normal file
View File

@ -0,0 +1,87 @@
import extractExtras from '../src/extractExtras';
describe('extractExtras', () => {
const baseQueryFormData = {
datasource: '1__table',
granularity_sqla: 'ds',
time_grain_sqla: 'PT1M',
viz_type: 'my_viz',
filters: [
{
col: 'gender',
op: '=',
val: 'girl',
},
],
};
it('should override formData with double underscored date options', () => {
expect(
extractExtras({
...baseQueryFormData,
extra_filters: [
{
col: '__time_col',
op: '=',
val: 'ds2',
},
{
col: '__time_grain',
op: '=',
val: 'PT5M',
},
{
col: '__time_range',
op: '=',
val: '2009-07-17T00:00:00 : 2020-07-17T00:00:00',
},
],
}),
).toEqual({
extras: {
time_grain_sqla: 'PT5M',
},
filters: [
{
col: 'gender',
op: '=',
val: 'girl',
},
],
granularity: 'ds2',
time_range: '2009-07-17T00:00:00 : 2020-07-17T00:00:00',
});
});
it('should create regular filters from non-reserved columns', () => {
expect(
extractExtras({
...baseQueryFormData,
extra_filters: [
{
col: 'name',
op: 'IN',
val: ['Eve', 'Evelyn'],
},
],
}),
).toEqual({
extras: {
time_grain_sqla: 'PT1M',
},
filters: [
{
col: 'gender',
op: '=',
val: 'girl',
},
{
col: 'name',
op: 'IN',
val: ['Eve', 'Evelyn'],
},
],
granularity: 'ds',
});
});
});

39
superset-frontend/temporary_superset_ui/superset-ui/packages/superset-ui-query/test/processFilters.test.ts
View File

@ -14,6 +14,7 @@ describe('processFilters', () => {
it('should handle an empty array', () => {
expect(
processFilters({
where: '1 = 1',
granularity: 'something',
viz_type: 'custom',
datasource: 'boba',
@ -21,9 +22,11 @@ describe('processFilters', () => {
}),
).toEqual({
filters: [],
having: '',
having_filters: [],
where: '',
extras: {
having: '',
having_druid: [],
where: '(1 = 1)',
},
});
});
@ -84,6 +87,22 @@ describe('processFilters', () => {
],
}),
).toEqual({
extras: {
having: '(ice = 25 OR ice = 50) AND (waitTime <= 180)',
having_druid: [
{
col: 'sweetness',
op: '>',
val: '0',
},
{
col: 'sweetness',
op: '<=',
val: '50',
},
],
where: '(tea = "jasmine") AND (cup = "large")',
},
filters: [
{
col: 'milk',
@ -95,20 +114,6 @@ describe('processFilters', () => {
val: 'almond',
},
],
having: '(ice = 25 OR ice = 50) AND (waitTime <= 180)',
having_filters: [
{
col: 'sweetness',
op: '>',
val: '0',
},
{
col: 'sweetness',
op: '<=',
val: '50',
},
],
where: '(tea = "jasmine") AND (cup = "large")',
});
});
});

12
superset-frontend/temporary_superset_ui/superset-ui/plugins/plugin-chart-word-cloud/src/plugin/buildQuery.ts
View File

@ -3,9 +3,11 @@ import { WordCloudFormData } from '../types';
export default function buildQuery(formData: WordCloudFormData) {
// Set the single QueryObject's groupby field with series in formData
return buildQueryContext(formData, baseQueryObject => [
{
...baseQueryObject,
},
]);
return buildQueryContext(formData, baseQueryObject => {
return [
{
...baseQueryObject,
},
];
});
}