pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(plugins): BigNumber Time Comparison with existing time_offset API (#27718) 

Co-authored-by: lilykuang <jialikuang@gmail.com>
Co-authored-by: Kamil Gabryjelski <kamil.gabryjelski@gmail.com>
This commit is contained in:
Antonio Rivero 2024-05-16 18:47:50 +02:00 committed by GitHub
parent b69958b412
commit b1f85dce71
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1533 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
superset-frontend/packages/superset-ui-chart-controls/src/sections/index.ts
View File

@ -23,3 +23,4 @@ export * from './annotationsAndLayers';
export * from './forecastInterval';
export * from './chartTitle';
export * from './echartsTimeSeriesQuery';
export * from './timeComparison';

136
superset-frontend/packages/superset-ui-chart-controls/src/sections/timeComparison.tsx Normal file
View File

@ -0,0 +1,136 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { t, ComparisonType } from '@superset-ui/core';
import { ControlPanelSectionConfig } from '../types';
const fullChoices = [
['1 day ago', t('1 day ago')],
['1 week ago', t('1 week ago')],
['28 days ago', t('28 days ago')],
['30 days ago', t('30 days ago')],
['1 month ago', t('1 month ago')],
['52 weeks ago', t('52 weeks ago')],
['1 year ago', t('1 year ago')],
['104 weeks ago', t('104 weeks ago')],
['2 years ago', t('2 years ago')],
['156 weeks ago', t('156 weeks ago')],
['3 years ago', t('3 years ago')],
['custom', t('Custom date')],
['inherit', t('Inherit range from time filter')],
];
const reducedKeys = new Set([
'1 day ago',
'1 week ago',
'1 month ago',
'1 year ago',
'custom',
'inherit',
]);
// Filter fullChoices to get only the entries whose keys are in reducedKeys
const reducedChoices = fullChoices.filter(choice => reducedKeys.has(choice[0]));
type TimeComparisonControlsType = {
multi?: boolean;
showCalculationType?: boolean;
showFullChoices?: boolean;
};
export const timeComparisonControls: ({
multi,
showCalculationType,
showFullChoices,
}: TimeComparisonControlsType) => ControlPanelSectionConfig = ({
multi = true,
showCalculationType = true,
showFullChoices = true,
}) => ({
label: t('Time Comparison'),
tabOverride: 'data',
description: t('Compare results with other time periods.'),
controlSetRows: [
[
{
name: 'time_compare',
config: {
type: 'SelectControl',
multi,
freeForm: true,
placeholder: t('Select or type a custom value...'),
label: t('Time shift'),
choices: showFullChoices ? fullChoices : reducedChoices,
description: t(
'Overlay results from a relative time period. ' +
'Expects relative time deltas ' +
'in natural language (example: 24 hours, 7 days, ' +
'52 weeks, 365 days). Free text is supported. ' +
'Use "Inherit range from time filters" ' +
'to shift the comparison time range ' +
'by the same length as your time range ' +
'and use "Custom" to set a custom comparison range.',
),
},
},
],
[
{
name: 'start_date_offset',
config: {
type: 'TimeOffsetControl',
label: t('shift start date'),
visibility: ({ controls }) =>
controls?.time_compare.value === 'custom',
},
},
],
[
{
name: 'comparison_type',
config: {
type: 'SelectControl',
label: t('Calculation type'),
default: 'values',
choices: [
[ComparisonType.Values, t('Actual values')],
[ComparisonType.Difference, t('Difference')],
[ComparisonType.Percentage, t('Percentage change')],
[ComparisonType.Ratio, t('Ratio')],
],
description: t(
'How to display time shifts: as individual lines; as the ' +
'difference between the main time series and each time shift; ' +
'as the percentage change; or as the ratio between series and time shifts.',
),
visibility: () => Boolean(showCalculationType),
},
},
],
[
{
name: 'comparison_range_label',
config: {
type: 'ComparisonRangeLabel',
multi,
visibility: ({ controls }) => Boolean(controls?.time_compare.value),
},
},
],
],
});

63
superset-frontend/packages/superset-ui-core/src/time-comparison/fetchTimeRange.ts
View File

@ -17,7 +17,12 @@
* under the License.
*/
import rison from 'rison';
import { SupersetClient, getClientErrorObject } from '@superset-ui/core';
import { isEmpty } from 'lodash';
import {
SupersetClient,
getClientErrorObject,
ensureIsArray,
} from '@superset-ui/core';
export const SEPARATOR = ' : ';
@ -39,20 +44,64 @@ export const formatTimeRange = (
)} ≤ ${columnPlaceholder} < ${formatDateEndpoint(splitDateRange[1])}`;
};
export const formatTimeRangeComparison = (
initialTimeRange: string,
shiftedTimeRange: string,
columnPlaceholder = 'col',
) => {
const splitInitialDateRange = initialTimeRange.split(SEPARATOR);
const splitShiftedDateRange = shiftedTimeRange.split(SEPARATOR);
return `${columnPlaceholder}: ${formatDateEndpoint(
splitInitialDateRange[0],
true,
)} to ${formatDateEndpoint(splitInitialDateRange[1])} vs
${formatDateEndpoint(splitShiftedDateRange[0], true)} to ${formatDateEndpoint(
splitShiftedDateRange[1],
)}`;
};
export const fetchTimeRange = async (
timeRange: string,
columnPlaceholder = 'col',
shifts?: string[],
) => {
const query = rison.encode_uri(timeRange);
const endpoint = `/api/v1/time_range/?q=${query}`;
let query;
let endpoint;
if (!isEmpty(shifts)) {
const timeRanges = ensureIsArray(shifts).map(shift => ({
timeRange,
shift,
}));
query = rison.encode_uri([{ timeRange }, ...timeRanges]);
endpoint = `/api/v1/time_range/?q=${query}`;
} else {
query = rison.encode_uri(timeRange);
endpoint = `/api/v1/time_range/?q=${query}`;
}
try {
const response = await SupersetClient.get({ endpoint });
const timeRangeString = buildTimeRangeString(
response?.json?.result[0]?.since || '',
response?.json?.result[0]?.until || '',
if (isEmpty(shifts)) {
const timeRangeString = buildTimeRangeString(
response?.json?.result[0]?.since || '',
response?.json?.result[0]?.until || '',
);
return {
value: formatTimeRange(timeRangeString, columnPlaceholder),
};
}
const timeRanges = response?.json?.result.map((result: any) =>
buildTimeRangeString(result.since, result.until),
);
return {
value: formatTimeRange(timeRangeString, columnPlaceholder),
value: timeRanges
.slice(1)
.map((timeRange: string) =>
formatTimeRangeComparison(
timeRanges[0],
timeRange,
columnPlaceholder,
),
),
};
} catch (response) {
const clientError = await getClientErrorObject(response);

121
superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts Normal file
View File

@ -0,0 +1,121 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { ensureIsArray } from '../utils';
const DAY_IN_MS = 24 * 60 * 60 * 1000;
export const parseDttmToDate = (dttm: string): Date => {
const now = new Date();
now.setUTCHours(0, 0, 0, 0);
if (dttm === 'now' || dttm === 'today' || dttm === 'No filter') {
return now;
}
if (dttm === 'Last week') {
now.setUTCDate(now.getUTCDate() - 7);
return now;
}
if (dttm === 'Last month') {
now.setUTCMonth(now.getUTCMonth() - 1);
now.setUTCDate(1);
return now;
}
if (dttm === 'Last quarter') {
now.setUTCMonth(now.getUTCMonth() - 3);
now.setUTCDate(1);
return now;
}
if (dttm === 'Last year') {
now.setUTCFullYear(now.getUTCFullYear() - 1);
now.setUTCDate(1);
return now;
}
if (dttm === 'previous calendar week') {
now.setUTCDate(now.getUTCDate() - now.getUTCDay());
return now;
}
if (dttm === 'previous calendar month') {
now.setUTCMonth(now.getUTCMonth() - 1, 1);
return now;
}
if (dttm === 'previous calendar year') {
now.setUTCFullYear(now.getUTCFullYear() - 1, 0, 1);
return now;
}
if (dttm?.includes('ago')) {
const parts = dttm.split(' ');
const amount = parseInt(parts[0], 10);
const unit = parts[1];
switch (unit) {
case 'day':
case 'days':
now.setUTCDate(now.getUTCDate() - amount);
break;
case 'week':
case 'weeks':
now.setUTCDate(now.getUTCDate() - amount * 7);
break;
case 'month':
case 'months':
now.setUTCMonth(now.getUTCMonth() - amount);
break;
case 'year':
case 'years':
now.setUTCFullYear(now.getUTCFullYear() - amount);
break;
default:
break;
}
return now;
}
const parsed = new Date(dttm);
parsed.setUTCHours(0, 0, 0, 0);
return parsed;
};
export const getTimeOffset = (
timeRangeFilter: any,
shifts: string[],
startDate: string,
): string[] => {
const isCustom = shifts?.includes('custom');
const isInherit = shifts?.includes('inherit');
const customStartDate = isCustom && parseDttmToDate(startDate).getTime();
const filterStartDate = parseDttmToDate(
timeRangeFilter.comparator.split(' : ')[0],
).getTime();
const filterEndDate = parseDttmToDate(
timeRangeFilter.comparator.split(' : ')[1],
).getTime();
const customShift =
customStartDate &&
Math.ceil((filterStartDate - customStartDate) / DAY_IN_MS);
const inInheritShift =
isInherit && Math.ceil((filterEndDate - filterStartDate) / DAY_IN_MS);
let newShifts = shifts;
if (isCustom) {
newShifts = [`${customShift} days ago`];
}
if (isInherit) {
newShifts = [`${inInheritShift} days ago`];
}
return ensureIsArray(newShifts);
};

1
superset-frontend/packages/superset-ui-core/src/time-comparison/index.ts
View File

@ -21,4 +21,5 @@ export * from './types';
export { default as getComparisonInfo } from './getComparisonInfo';
export { default as getComparisonFilters } from './getComparisonFilters';
export { parseDttmToDate, getTimeOffset } from './getTimeOffset';
export { SEPARATOR, fetchTimeRange } from './fetchTimeRange';

62
superset-frontend/packages/superset-ui-core/test/time-comparison/fetchTimeRange.test.ts
View File

@ -22,11 +22,12 @@ import { fetchTimeRange } from '@superset-ui/core';
import {
buildTimeRangeString,
formatTimeRange,
formatTimeRangeComparison,
} from '../../src/time-comparison/fetchTimeRange';
afterEach(fetchMock.restore);
it('generates proper time range string', () => {
test('generates proper time range string', () => {
expect(
buildTimeRangeString('2010-07-30T00:00:00', '2020-07-30T00:00:00'),
).toBe('2010-07-30T00:00:00 : 2020-07-30T00:00:00');
@ -36,7 +37,7 @@ it('generates proper time range string', () => {
expect(buildTimeRangeString('', '')).toBe(' : ');
});
it('generates a readable time range', () => {
test('generates a readable time range', () => {
expect(formatTimeRange('Last 7 days')).toBe('Last 7 days');
expect(formatTimeRange('No filter')).toBe('No filter');
expect(formatTimeRange('Yesterday : Tomorrow')).toBe(
@ -53,7 +54,7 @@ it('generates a readable time range', () => {
);
});
it('returns a formatted time range from response', async () => {
test('returns a formatted time range from response', async () => {
fetchMock.get("glob:*/api/v1/time_range/?q='Last+day'", {
result: [
{
@ -70,7 +71,7 @@ it('returns a formatted time range from response', async () => {
});
});
it('returns a formatted time range from empty response', async () => {
test('returns a formatted time range from empty response', async () => {
fetchMock.get("glob:*/api/v1/time_range/?q='Last+day'", {
result: [],
});
@ -81,7 +82,7 @@ it('returns a formatted time range from empty response', async () => {
});
});
it('returns a formatted error message from response', async () => {
test('returns a formatted error message from response', async () => {
fetchMock.getOnce("glob:*/api/v1/time_range/?q='Last+day'", {
throws: new Response(JSON.stringify({ message: 'Network error' })),
});
@ -116,3 +117,54 @@ it('returns a formatted error message from response', async () => {
error: 'Network error',
});
});
test('fetchTimeRange with shift', async () => {
fetchMock.getOnce(
"glob:*/api/v1/time_range/?q=!((timeRange:'Last+day'),(shift%3A'last%20month'%2CtimeRange%3A'Last%20day'))",
{
result: [
{
since: '2021-04-13T00:00:00',
until: '2021-04-14T00:00:00',
timeRange: 'Last day',
shift: null,
},
{
since: '2021-03-13T00:00:00',
until: '2021-03-14T00:00:00',
timeRange: 'Last day',
shift: 'last month',
},
],
},
);
const timeRange = await fetchTimeRange('Last day', 'temporal_col', [
'last month',
]);
expect(timeRange).toEqual({
value: [
'temporal_col: 2021-04-13 to 2021-04-14 vs\n 2021-03-13 to 2021-03-14',
],
});
});
test('formatTimeRangeComparison', () => {
expect(
formatTimeRangeComparison(
'2021-04-13T00:00:00 : 2021-04-14T00:00:00',
'2021-03-13T00:00:00 : 2021-03-14T00:00:00',
),
).toEqual('col: 2021-04-13 to 2021-04-14 vs\n 2021-03-13 to 2021-03-14');
expect(
formatTimeRangeComparison(
'2021-04-13T00:00:00 : 2021-04-14T00:00:00',
'2021-03-13T00:00:00 : 2021-03-14T00:00:00',
'col_name',
),
).toEqual(
'col_name: 2021-04-13 to 2021-04-14 vs\n 2021-03-13 to 2021-03-14',
);
});

46
superset-frontend/packages/superset-ui-core/test/time-comparison/getTimeOffset.test.ts Normal file
View File

@ -0,0 +1,46 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { getTimeOffset } from '@superset-ui/core';
test('handles custom shifts', () => {
const shifts = ['custom'];
const startDate = '2023-01-01';
const timeRangeFilter = { comparator: '2023-01-03 : 2023-01-10' };
const result = getTimeOffset(timeRangeFilter, shifts, startDate);
expect(result).toEqual(['2 days ago']);
});
test('handles inherit shifts', () => {
const shifts = ['inherit'];
const startDate = '';
const timeRangeFilter = { comparator: '2023-01-03 : 2023-01-10' };
const result = getTimeOffset(timeRangeFilter, shifts, startDate);
expect(result).toEqual(['7 days ago']);
});
test('handles no custom or inherit shifts', () => {
const shifts = ['1 week ago'];
const startDate = '';
const timeRangeFilter = { comparator: '2023-01-03 : 2023-01-10' };
const result = getTimeOffset(timeRangeFilter, shifts, startDate);
expect(result).toEqual(['1 week ago']);
});

131
superset-frontend/packages/superset-ui-core/test/time-comparison/parseDttmToDate.test.ts Normal file
View File

@ -0,0 +1,131 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { parseDttmToDate } from '@superset-ui/core';
test('should handle "now"', () => {
const now = parseDttmToDate('now');
const expected = new Date();
expected.setUTCHours(0, 0, 0, 0);
expect(expected).toEqual(now);
});
test('should handle "today" and "No filter"', () => {
const today = parseDttmToDate('today');
const noFilter = parseDttmToDate('No filter');
const expected = new Date();
expected.setUTCHours(0, 0, 0, 0);
expect(today).toEqual(expected);
expect(noFilter).toEqual(expected);
});
test('should handle relative time strings', () => {
const lastWeek = parseDttmToDate('Last week');
const lastMonth = parseDttmToDate('Last month');
const lastQuarter = parseDttmToDate('Last quarter');
const lastYear = parseDttmToDate('Last year');
let now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCDate(now.getUTCDate() - 7);
expect(lastWeek).toEqual(now);
now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCMonth(now.getUTCMonth() - 1);
now.setUTCDate(1);
expect(lastMonth).toEqual(now);
now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCMonth(now.getUTCMonth() - 3);
now.setUTCDate(1);
expect(lastQuarter).toEqual(now);
now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCFullYear(now.getUTCFullYear() - 1);
now.setUTCDate(1);
expect(lastYear).toEqual(now);
});
test('should handle previous calendar units', () => {
let now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCDate(now.getUTCDate() - now.getUTCDay());
const previousWeek = parseDttmToDate('previous calendar week');
expect(previousWeek).toEqual(now);
now = new Date();
now.setUTCMonth(now.getUTCMonth() - 1, 1);
now.setUTCHours(0, 0, 0, 0);
const previousMonth = parseDttmToDate('previous calendar month');
expect(previousMonth).toEqual(now);
now = new Date();
now.setUTCFullYear(now.getUTCFullYear() - 1, 0, 1);
now.setUTCHours(0, 0, 0, 0);
const previousYear = parseDttmToDate('previous calendar year');
expect(previousYear).toEqual(now);
});
test('should handle dynamic "ago" times', () => {
const fiveDaysAgo = parseDttmToDate('5 days ago');
const fiveDayAgo = parseDttmToDate('5 day ago');
let now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCDate(now.getUTCDate() - 5);
expect(fiveDaysAgo).toEqual(now);
expect(fiveDayAgo).toEqual(now);
const weeksAgo = parseDttmToDate('7 weeks ago');
const weekAgo = parseDttmToDate('7 week ago');
now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCDate(now.getUTCDate() - 7 * 7);
expect(weeksAgo).toEqual(now);
expect(weekAgo).toEqual(now);
const fiveMonthsAgo = parseDttmToDate('5 months ago');
const fiveMonthAgo = parseDttmToDate('5 month ago');
now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCMonth(now.getUTCMonth() - 5);
expect(fiveMonthsAgo).toEqual(now);
expect(fiveMonthAgo).toEqual(now);
const fiveYearsAgo = parseDttmToDate('5 years ago');
const fiveYearAgo = parseDttmToDate('5 year ago');
now = new Date();
now.setUTCHours(0, 0, 0, 0);
now.setUTCFullYear(now.getUTCFullYear() - 5);
expect(fiveYearsAgo).toEqual(now);
expect(fiveYearAgo).toEqual(now);
// default case
const fiveHoursAgo = parseDttmToDate('5 hours ago');
now = new Date();
now.setUTCHours(0, 0, 0, 0);
expect(fiveHoursAgo).toEqual(now);
});
test('should parse valid moment strings', () => {
const specificDate = new Date('2023-01-01');
specificDate.setUTCHours(0, 0, 0, 0);
const parsedDate = parseDttmToDate('2023-01-01');
expect(parsedDate).toEqual(specificDate);
});

48
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/PopKPI.tsx
View File

@ -16,9 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
import React, { useMemo } from 'react';
import { css, styled, t, useTheme } from '@superset-ui/core';
import React, { useEffect, useMemo, useState } from 'react';
import {
css,
ensureIsArray,
fetchTimeRange,
getTimeOffset,
styled,
t,
useTheme,
} from '@superset-ui/core';
import { Tooltip } from '@superset-ui/chart-controls';
import { isEmpty } from 'lodash';
import {
ColorSchemeEnum,
PopKPIComparisonSymbolStyleProps,
@ -69,9 +78,38 @@ export default function PopKPI(props: PopKPIProps) {
comparisonColorEnabled,
comparisonColorScheme,
percentDifferenceNumber,
comparatorText,
currentTimeRangeFilter,
startDateOffset,
shift,
} = props;
const [comparisonRange, setComparisonRange] = useState<string>('');
useEffect(() => {
if (!currentTimeRangeFilter || (!shift && !startDateOffset)) {
setComparisonRange('');
} else if (!isEmpty(shift) || startDateOffset) {
const newShift = getTimeOffset(
currentTimeRangeFilter,
ensureIsArray(shift),
startDateOffset || '',
);
const promise: any = fetchTimeRange(
(currentTimeRangeFilter as any).comparator,
currentTimeRangeFilter.subject,
newShift || [],
);
Promise.resolve(promise).then((res: any) => {
const response: string[] = ensureIsArray(res.value);
const firstRange: string = response.flat()[0];
const rangeText = firstRange.split('vs\n');
setComparisonRange(
rangeText.length > 1 ? rangeText[1].trim() : rangeText[0],
);
});
}
}, [currentTimeRangeFilter, shift, startDateOffset]);
const theme = useTheme();
const flexGap = theme.gridUnit * 5;
const wrapperDivStyles = css`
@ -150,7 +188,7 @@ export default function PopKPI(props: PopKPIProps) {
{
symbol: '#',
value: prevNumber,
tooltipText: t('Data for %s', comparatorText),
tooltipText: t('Data for %s', comparisonRange || 'previous range'),
},
{
symbol: '△',
@ -164,7 +202,7 @@ export default function PopKPI(props: PopKPIProps) {
},
],
[
comparatorText,
comparisonRange,
prevNumber,
valueDifference,
percentDifferenceFormattedString,

64
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts
View File

@ -18,50 +18,50 @@
*/
import {
buildQueryContext,
getComparisonInfo,
ComparisonTimeRangeType,
QueryFormData,
PostProcessingRule,
ensureIsArray,
SimpleAdhocFilter,
getTimeOffset,
} from '@superset-ui/core';
import {
isTimeComparison,
timeCompareOperator,
} from '@superset-ui/chart-controls';
export default function buildQuery(formData: QueryFormData) {
const {
cols: groupby,
time_comparison: timeComparison,
extra_form_data: extraFormData,
} = formData;
const { cols: groupby } = formData;
const queryContextA = buildQueryContext(formData, baseQueryObject => [
{
...baseQueryObject,
groupby,
},
]);
const queryContextA = buildQueryContext(formData, baseQueryObject => {
const postProcessing: PostProcessingRule[] = [];
postProcessing.push(timeCompareOperator(formData, baseQueryObject));
const TimeRangeFilters =
formData.adhoc_filters?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
const comparisonFormData = getComparisonInfo(
formData,
timeComparison,
extraFormData,
);
const queryContextB = buildQueryContext(
comparisonFormData,
baseQueryObject => [
const timeOffsets = ensureIsArray(
isTimeComparison(formData, baseQueryObject)
? getTimeOffset(
TimeRangeFilters[0],
formData.time_compare,
formData.start_date_offset,
)
: [],
);
return [
{
...baseQueryObject,
groupby,
extras: {
...baseQueryObject.extras,
instant_time_comparison_range:
timeComparison !== ComparisonTimeRangeType.Custom
? timeComparison
: undefined,
},
post_processing: postProcessing,
time_offsets: isTimeComparison(formData, baseQueryObject)
? ensureIsArray(timeOffsets)
: [],
},
],
);
];
});
return {
...queryContextA,
queries: [...queryContextA.queries, ...queryContextB.queries],
};
}

84
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/controlPanel.ts
View File

@ -16,20 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
import { t } from '@superset-ui/core';
import {
AdhocFilter,
ComparisonTimeRangeType,
SimpleAdhocFilter,
t,
validateTimeComparisonRangeValues,
} from '@superset-ui/core';
import {
ColumnMeta,
ControlPanelConfig,
ControlPanelState,
ControlState,
getStandardizedControls,
sharedControls,
sections,
} from '@superset-ui/chart-controls';
import { headerFontSize, subheaderFontSize } from '../sharedControls';
import { ColorSchemeEnum } from './types';
@ -42,70 +34,6 @@ const config: ControlPanelConfig = {
controlSetRows: [
['metric'],
['adhoc_filters'],
[
{
name: 'time_comparison',
config: {
type: 'SelectControl',
label: t('Range for Comparison'),
default: 'r',
choices: [
['r', 'Inherit range from time filters'],
['y', 'Year'],
['m', 'Month'],
['w', 'Week'],
['c', 'Custom'],
],
rerender: ['adhoc_custom'],
description: t(
'Set the time range that will be used for the comparison metrics. ' +
'For example, "Year" will compare to the same dates one year earlier. ' +
'Use "Inherit range from time filters" to shift the comparison time range' +
'by the same length as your time range and use "Custom" to set a custom comparison range.',
),
},
},
],
[
{
name: `adhoc_custom`,
config: {
...sharedControls.adhoc_filters,
label: t('Filters for Comparison'),
description:
'This only applies when selecting the Range for Comparison Type: Custom',
visibility: ({ controls }) =>
controls?.time_comparison?.value ===
ComparisonTimeRangeType.Custom,
mapStateToProps: (
state: ControlPanelState,
controlState: ControlState,
) => {
const originalMapStateToPropsRes =
sharedControls.adhoc_filters.mapStateToProps?.(
state,
controlState,
) || {};
const columns = originalMapStateToPropsRes.columns.filter(
(col: ColumnMeta) =>
col.is_dttm &&
(state.controls.adhoc_filters.value as AdhocFilter[]).some(
(val: SimpleAdhocFilter) =>
val.subject === col.column_name,
),
);
return {
...originalMapStateToPropsRes,
columns,
externalValidationErrors: validateTimeComparisonRangeValues(
state.controls?.time_comparison?.value,
controlState.value,
),
};
},
},
},
],
[
{
name: 'row_limit',
@ -180,14 +108,16 @@ const config: ControlPanelConfig = {
],
],
},
sections.timeComparisonControls({
multi: false,
showCalculationType: false,
showFullChoices: false,
}),
],
controlOverrides: {
y_axis_format: {
label: t('Number format'),
},
adhoc_filters: {
rerender: ['adhoc_custom'],
},
},
formDataOverrides: formData => ({
...formData,

58
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts
View File

@ -22,7 +22,9 @@ import {
getMetricLabel,
getValueFormatter,
getNumberFormatter,
formatTimeRange,
SimpleAdhocFilter,
ensureIsArray,
getTimeOffset,
} from '@superset-ui/core';
import { getComparisonFontSize, getHeaderFontSize } from './utils';
@ -87,17 +89,49 @@ export default function transformProps(chartProps: ChartProps) {
percentDifferenceFormat,
} = formData;
const { data: dataA = [] } = queriesData[0];
const {
data: dataB = [],
from_dttm: comparisonFromDatetime,
to_dttm: comparisonToDatetime,
} = queriesData[1];
const data = dataA;
const metricName = getMetricLabel(metric);
const timeComparison = ensureIsArray(chartProps.rawFormData?.time_compare)[0];
const startDateOffset = chartProps.rawFormData?.start_date_offset;
const currentTimeRangeFilter = chartProps.rawFormData?.adhoc_filters?.filter(
(adhoc_filter: SimpleAdhocFilter) =>
adhoc_filter.operator === 'TEMPORAL_RANGE',
)?.[0];
const isCustomOrInherit =
timeComparison === 'custom' || timeComparison === 'inherit';
let dataOffset: string[] = [];
if (isCustomOrInherit) {
dataOffset = getTimeOffset(
currentTimeRangeFilter,
ensureIsArray(timeComparison),
startDateOffset || '',
);
}
const { value1, value2 } = data.reduce(
(acc: { value1: number; value2: number }, curr: { [x: string]: any }) => {
Object.keys(curr).forEach(key => {
if (
key.includes(
`${metricName}__${
!isCustomOrInherit ? timeComparison : dataOffset[0]
}`,
)
) {
acc.value2 += curr[key];
} else if (key.includes(metricName)) {
acc.value1 += curr[key];
}
});
return acc;
},
{ value1: 0, value2: 0 },
);
let bigNumber: number | string =
data.length === 0 ? 0 : parseMetricValue(data[0][metricName]);
data.length === 0 ? 0 : parseMetricValue(value1);
let prevNumber: number | string =
data.length === 0 ? 0 : parseMetricValue(dataB[0][metricName]);
data.length === 0 ? 0 : parseMetricValue(value2);
const numberFormatter = getValueFormatter(
metric,
@ -133,10 +167,6 @@ export default function transformProps(chartProps: ChartProps) {
prevNumber = numberFormatter(prevNumber);
valueDifference = numberFormatter(valueDifference);
const percentDifference: string = formatPercentChange(percentDifferenceNum);
const comparatorText = formatTimeRange('%Y-%m-%d', [
comparisonFromDatetime,
comparisonToDatetime,
]);
return {
width,
@ -155,6 +185,8 @@ export default function transformProps(chartProps: ChartProps) {
comparisonColorEnabled,
comparisonColorScheme,
percentDifferenceNumber: percentDifferenceNum,
comparatorText,
currentTimeRangeFilter,
startDateOffset,
shift: timeComparison,
};
}

5
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/types.ts
View File

@ -21,6 +21,7 @@ import {
supersetTheme,
TimeseriesDataRecord,
Metric,
SimpleAdhocFilter,
} from '@superset-ui/core';
export interface PopKPIStylesProps {
@ -60,8 +61,10 @@ export type PopKPIProps = PopKPIStylesProps &
percentDifferenceFormattedString: string;
compType: string;
percentDifferenceNumber: number;
comparatorText: string;
comparisonColorScheme?: string;
currentTimeRangeFilter?: SimpleAdhocFilter;
startDateOffset?: string;
shift: string;
};
export enum ColorSchemeEnum {

107
superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx Normal file
View File

@ -0,0 +1,107 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React, { useEffect, useState } from 'react';
import { useSelector } from 'react-redux';
import { isEmpty, isEqual } from 'lodash';
import {
BinaryAdhocFilter,
css,
ensureIsArray,
fetchTimeRange,
getTimeOffset,
SimpleAdhocFilter,
t,
} from '@superset-ui/core';
import ControlHeader, {
ControlHeaderProps,
} from 'src/explore/components/ControlHeader';
import { RootState } from 'src/views/store';
const isTimeRangeEqual = (
left: BinaryAdhocFilter[],
right: BinaryAdhocFilter[],
) => isEqual(left, right);
type ComparisonRangeLabelProps = ControlHeaderProps & {
multi?: boolean;
};
export const ComparisonRangeLabel = ({
multi = true,
}: ComparisonRangeLabelProps) => {
const [labels, setLabels] = useState<string[]>([]);
const currentTimeRangeFilters = useSelector<RootState, BinaryAdhocFilter[]>(
state =>
state.explore.form_data.adhoc_filters.filter(
(adhoc_filter: SimpleAdhocFilter) =>
adhoc_filter.operator === 'TEMPORAL_RANGE',
),
isTimeRangeEqual,
);
const shifts = useSelector<RootState, string[]>(
state => state.explore.form_data.time_compare,
);
const startDate = useSelector<RootState, string>(
state => state.explore.form_data.start_date_offset,
);
useEffect(() => {
const shiftsArray = ensureIsArray(shifts);
if (
isEmpty(currentTimeRangeFilters) ||
(isEmpty(shiftsArray) && !startDate)
) {
setLabels([]);
} else if (!isEmpty(shifts) || startDate) {
const promises = currentTimeRangeFilters.map(filter => {
const newShifts = getTimeOffset(filter, shiftsArray, startDate);
return fetchTimeRange(
filter.comparator,
filter.subject,
ensureIsArray(newShifts),
);
});
Promise.all(promises).then(res => {
// access the value property inside the res and set the labels with it in the state
setLabels(res.map(r => r.value ?? ''));
});
}
}, [currentTimeRangeFilters, shifts, startDate]);
return labels.length ? (
<>
<ControlHeader label={t('Actual range for comparison')} />
{labels.flat().map(label => (
<>
<div
css={theme => css`
font-size: ${theme.typography.sizes.m}px;
color: ${theme.colors.grayscale.dark1};
`}
key={label}
>
{label}
</div>
</>
))}
</>
) : null;
};

87
superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx Normal file
View File

@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React, { ReactNode } from 'react';
import { isEqual } from 'lodash';
import moment, { Moment } from 'moment';
import {
parseDttmToDate,
BinaryAdhocFilter,
SimpleAdhocFilter,
css,
} from '@superset-ui/core';
import { DatePicker } from 'antd';
import { RangePickerProps } from 'antd/lib/date-picker';
import { useSelector } from 'react-redux';
import ControlHeader from 'src/explore/components/ControlHeader';
import { RootState } from 'src/views/store';
export interface TimeOffsetControlsProps {
label?: ReactNode;
startDate?: string;
description?: string;
hovered?: boolean;
value?: Moment;
onChange: (datetime: string) => void;
}
const MOMENT_FORMAT = 'YYYY-MM-DD';
const isTimeRangeEqual = (
left: BinaryAdhocFilter[],
right: BinaryAdhocFilter[],
) => isEqual(left, right);
export default function TimeOffsetControls({
onChange,
...props
}: TimeOffsetControlsProps) {
const currentTimeRangeFilters = useSelector<RootState, BinaryAdhocFilter[]>(
state =>
state.explore.form_data.adhoc_filters.filter(
(adhoc_filter: SimpleAdhocFilter) =>
adhoc_filter.operator === 'TEMPORAL_RANGE',
),
isTimeRangeEqual,
);
const startDate = currentTimeRangeFilters[0]?.comparator.split(' : ')[0];
const formatedDate = moment(parseDttmToDate(startDate));
const disabledDate: RangePickerProps['disabledDate'] = current =>
current && current > formatedDate;
return (
<div>
<ControlHeader {...props} />
<DatePicker
css={css`
width: 100%;
`}
onChange={(datetime: Moment) =>
onChange(datetime ? datetime.format(MOMENT_FORMAT) : '')
}
defaultPickerValue={
startDate ? moment(formatedDate).subtract(1, 'day') : undefined
}
disabledDate={disabledDate}
defaultValue={formatedDate}
/>
</div>
);
}

4
superset-frontend/src/explore/components/controls/index.js
View File

@ -34,6 +34,7 @@ import SpatialControl from './SpatialControl';
import TextAreaControl from './TextAreaControl';
import TextControl from './TextControl';
import TimeSeriesColumnControl from './TimeSeriesColumnControl';
import TimeOffsetControl from './TimeOffsetControl';
import ViewportControl from './ViewportControl';
import VizTypeControl from './VizTypeControl';
import MetricsControl from './MetricControl/MetricsControl';
@ -48,6 +49,7 @@ import DndColumnSelectControl, {
import XAxisSortControl from './XAxisSortControl';
import CurrencyControl from './CurrencyControl';
import ColumnConfigControl from './ColumnConfigControl';
import { ComparisonRangeLabel } from './ComparisonRangeLabel';
const controlMap = {
AnnotationLayerControl,
@ -80,6 +82,8 @@ const controlMap = {
ConditionalFormattingControl,
XAxisSortControl,
ContourControl,
ComparisonRangeLabel,
TimeOffsetControl,
...sharedControlComponents,
};
export default controlMap;

115
superset/common/query_context_processor.py
View File

@ -19,7 +19,7 @@ from __future__ import annotations
import copy
import logging
import re
from typing import Any, ClassVar, TYPE_CHECKING, TypedDict
from typing import Any, cast, ClassVar, TYPE_CHECKING, TypedDict
import numpy as np
import pandas as pd
@ -55,6 +55,7 @@ from superset.utils.core import (
DateColumn,
DTTM_ALIAS,
error_msg_from_exception,
FilterOperator,
get_base_axis_labels,
get_column_names_from_columns,
get_column_names_from_metrics,
@ -390,11 +391,6 @@ class QueryContextProcessor:
time_grain = self.get_time_grain(query_object)
if not time_grain:
raise QueryObjectValidationError(
_("Time Grain must be specified when using Time Shift.")
)
metric_names = get_metric_names(query_object.metrics)
# use columns that are not metrics as join keys
@ -429,6 +425,28 @@ class QueryContextProcessor:
query_object_clone.inner_to_dttm = outer_to_dttm
query_object_clone.time_offsets = []
query_object_clone.post_processing = []
# Get time offset index
index = (get_base_axis_labels(query_object.columns) or [DTTM_ALIAS])[0]
# The comparison is not using a temporal column so we need to modify
# the temporal filter so we run the query with the correct time range
if not dataframe_utils.is_datetime_series(df.get(index)):
# Lets find the first temporal filter in the filters array and change
# its val to be the result of get_since_until with the offset
for flt in query_object_clone.filter:
if flt.get(
"op"
) == FilterOperator.TEMPORAL_RANGE.value and isinstance(
flt.get("val"), str
):
time_range = cast(str, flt.get("val"))
(
new_outer_from_dttm,
new_outer_to_dttm,
) = get_since_until_from_time_range(
time_range=time_range,
time_shift=offset,
)
flt["val"] = f"{new_outer_from_dttm} : {new_outer_to_dttm}"
query_object_clone.filter = [
flt
for flt in query_object_clone.filter
@ -488,16 +506,6 @@ class QueryContextProcessor:
# 2. rename extra query columns
offset_metrics_df = offset_metrics_df.rename(columns=metrics_mapping)
# 3. set time offset for index
index = (get_base_axis_labels(query_object.columns) or [DTTM_ALIAS])[0]
if not dataframe_utils.is_datetime_series(offset_metrics_df.get(index)):
raise QueryObjectValidationError(
_(
"A time column must be specified "
"when using a Time Comparison."
)
)
# cache df and query
value = {
"df": offset_metrics_df,
@ -526,7 +534,7 @@ class QueryContextProcessor:
self,
df: pd.DataFrame,
offset_dfs: dict[str, pd.DataFrame],
time_grain: str,
time_grain: str | None,
join_keys: list[str],
) -> pd.DataFrame:
"""
@ -541,43 +549,58 @@ class QueryContextProcessor:
time_grain
)
if join_column_producer and not time_grain:
raise QueryObjectValidationError(
_("Time Grain must be specified when using Time Shift.")
)
# iterate on offset_dfs, left join each with df
for offset, offset_df in offset_dfs.items():
# defines a column name for the offset join column
column_name = OFFSET_JOIN_COLUMN_SUFFIX + offset
actual_join_keys = join_keys
# add offset join column to df
self.add_offset_join_column(
df, column_name, time_grain, offset, join_column_producer
)
if time_grain:
# defines a column name for the offset join column
column_name = OFFSET_JOIN_COLUMN_SUFFIX + offset
# add offset join column to offset_df
self.add_offset_join_column(
offset_df, column_name, time_grain, None, join_column_producer
)
# add offset join column to df
self.add_offset_join_column(
df, column_name, time_grain, offset, join_column_producer
)
# the temporal column is the first column in the join keys
# so we use the join column instead of the temporal column
actual_join_keys = [column_name, *join_keys[1:]]
# add offset join column to offset_df
self.add_offset_join_column(
offset_df, column_name, time_grain, None, join_column_producer
)
# left join df with offset_df
df = dataframe_utils.left_join_df(
left_df=df,
right_df=offset_df,
join_keys=actual_join_keys,
rsuffix=R_SUFFIX,
)
# the temporal column is the first column in the join keys
# so we use the join column instead of the temporal column
actual_join_keys = [column_name, *join_keys[1:]]
# move the temporal column to the first column in df
col = df.pop(join_keys[0])
df.insert(0, col.name, col)
if join_keys:
df = dataframe_utils.left_join_df(
left_df=df,
right_df=offset_df,
join_keys=actual_join_keys,
rsuffix=R_SUFFIX,
)
else:
df = dataframe_utils.full_outer_join_df(
left_df=df,
right_df=offset_df,
rsuffix=R_SUFFIX,
)
# removes columns created only for join purposes
df.drop(
list(df.filter(regex=f"{OFFSET_JOIN_COLUMN_SUFFIX}|{R_SUFFIX}")),
axis=1,
inplace=True,
)
if time_grain:
# move the temporal column to the first column in df
col = df.pop(join_keys[0])
df.insert(0, col.name, col)
# removes columns created only for join purposes
df.drop(
list(df.filter(regex=f"{OFFSET_JOIN_COLUMN_SUFFIX}|{R_SUFFIX}")),
axis=1,
inplace=True,
)
return df
@staticmethod

11
superset/common/utils/dataframe_utils.py
View File

@ -40,6 +40,17 @@ def left_join_df(
return df
def full_outer_join_df(
left_df: pd.DataFrame,
right_df: pd.DataFrame,
lsuffix: str = "",
rsuffix: str = "",
) -> pd.DataFrame:
df = left_df.join(right_df, lsuffix=lsuffix, rsuffix=rsuffix, how="outer")
df.reset_index(inplace=True)
return df
def df_metrics_to_num(df: pd.DataFrame, query_object: QueryObject) -> None:
"""Converting metrics to numeric when pandas.read_sql cannot"""
for col, dtype in df.dtypes.items():

8
superset/common/utils/time_range_utils.py
View File

@ -21,7 +21,7 @@ from typing import Any, cast
from superset import app
from superset.common.query_object import QueryObject
from superset.utils.core import FilterOperator, get_xaxis_label
from superset.utils.core import FilterOperator
from superset.utils.date_parser import get_since_until
@ -66,10 +66,8 @@ def get_since_until_from_query_object(
time_range = None
for flt in query_object.filter:
if (
flt.get("op") == FilterOperator.TEMPORAL_RANGE.value
and flt.get("col") == get_xaxis_label(query_object.columns)
and isinstance(flt.get("val"), str)
if flt.get("op") == FilterOperator.TEMPORAL_RANGE.value and isinstance(
flt.get("val"), str
):
time_range = cast(str, flt.get("val"))

212
superset/migrations/versions/2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison.py Normal file
View File

@ -0,0 +1,212 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Update charts with old time comparison controls
Revision ID: f84fde59123a
Revises: 9621c6d56ffb
Create Date: 2024-05-10 18:02:38.891060
"""
import json
import logging
from copy import deepcopy
from datetime import datetime, timedelta
from hashlib import md5
from typing import Any
from alembic import op
from sqlalchemy import Column, Integer, or_, String, Text
from sqlalchemy.ext.declarative import declarative_base
from superset import db
from superset.migrations.shared.utils import paginated_update
from superset.utils.date_parser import get_since_until
# revision identifiers, used by Alembic.
revision = "f84fde59123a"
down_revision = "9621c6d56ffb"
logger = logging.getLogger(__name__)
Base = declarative_base()
class Slice(Base):
__tablename__ = "slices"
id = Column(Integer, primary_key=True)
params = Column(Text)
viz_type = Column(String(250))
time_map = {
"r": "inherit",
"y": "1 year ago",
"m": "1 month ago",
"w": "1 week ago",
"c": "custom",
}
def upgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]:
params = deepcopy(slice_params)
if "enable_time_comparison" in params:
# Remove enable_time_comparison
del params["enable_time_comparison"]
# Update time_comparison to time_compare
if "time_comparison" in params:
time_comp = params.pop("time_comparison")
params["time_compare"] = time_map.get(
time_comp, "inherit"
) # Default to 'inherit' if not found
# Add comparison_type
params["comparison_type"] = "values"
# Adjust adhoc_custom
if "adhoc_custom" in params and params["adhoc_custom"]:
adhoc = params["adhoc_custom"][0] # As there's always only one element
if adhoc["comparator"] != "No filter":
# Set start_date_offset in params, not in adhoc
start_date_offset, _ = get_since_until(adhoc["comparator"])
params["start_date_offset"] = start_date_offset.strftime("%Y-%m-%d")
# delete adhoc_custom
del params["adhoc_custom"]
return params
def upgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
for slc in paginated_update(
session.query(Slice).filter(
or_(Slice.viz_type == "pop_kpi", Slice.viz_type == "table")
)
):
try:
params = json.loads(slc.params)
updated_slice_params = upgrade_comparison_params(params)
slc.params = json.dumps(updated_slice_params)
except Exception as ex:
session.rollback()
logger.exception(
f"An error occurred: Upgrading params for slice {slc.id} failed."
f"You need to fix it before upgrading your DB."
)
raise Exception(f"An error occurred while upgrading slice: {ex}")
session.commit()
session.close()
def downgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]:
params = deepcopy(slice_params)
reverse_time_map = {
v: k for k, v in time_map.items()
} # Reverse the map from the upgrade function
# Add enable_time_comparison
params["enable_time_comparison"] = True
# Revert time_compare to time_comparison
if "time_compare" in params:
time_comp = params.pop("time_compare")
params["time_comparison"] = reverse_time_map.get(
time_comp, "r"
) # Default to 'r' if not found
# Remove comparison_type
if "comparison_type" in params:
del params["comparison_type"]
# Default adhoc_custom
adhoc_custom = [
{
"clause": "WHERE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "No filter",
"expressionType": "SIMPLE",
}
]
# Handle start_date_offset and adjust adhoc_custom if necessary
if "start_date_offset" in params:
start_date_offset = datetime.strptime(
params.pop("start_date_offset"), "%Y-%m-%d"
)
adhoc_filters = params.get("adhoc_filters", [])
temporal_range_filter = next(
(f for f in adhoc_filters if f["operator"] == "TEMPORAL_RANGE"), None
)
if temporal_range_filter:
since, until = get_since_until(temporal_range_filter["comparator"])
delta_days = (until - since).days
new_until_date = start_date_offset + timedelta(days=delta_days - 1)
comparator_str = f"{start_date_offset.strftime('%Y-%m-%d')} : {new_until_date.strftime('%Y-%m-%d')}"
# Generate filterOptionName
random_string = md5(comparator_str.encode("utf-8")).hexdigest()
filter_option_name = f"filter_{random_string}"
adhoc_custom[0] = {
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": comparator_str,
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": filter_option_name,
}
params["adhoc_custom"] = adhoc_custom
return params
def downgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
for slc in paginated_update(
session.query(Slice).filter(
Slice.viz_type == "pop_kpi" or Slice.viz_type == "table"
)
):
try:
params = json.loads(slc.params)
updated_slice_params = downgrade_comparison_params(params)
slc.params = json.dumps(updated_slice_params)
except Exception as ex:
session.rollback()
logger.exception(
f"An error occurred: Downgrading params for slice {slc.id} failed."
f"You need to fix it before downgrading your DB."
)
raise Exception(f"An error occurred while downgrading slice: {ex}")
session.commit()
session.close()

7
superset/utils/date_parser.py
View File

@ -263,9 +263,10 @@ def get_since_until( # pylint: disable=too-many-arguments,too-many-locals,too-m
)
if time_shift:
time_delta = parse_past_timedelta(time_shift)
_since = _since if _since is None else (_since - time_delta)
_until = _until if _until is None else (_until - time_delta)
time_delta_since = parse_past_timedelta(time_shift, _since)
time_delta_until = parse_past_timedelta(time_shift, _until)
_since = _since if _since is None else (_since - time_delta_since)
_until = _until if _until is None else (_until - time_delta_until)
if instant_time_comparison_range:
# This is only set using the new time comparison controls

7
superset/views/api.py
View File

@ -46,6 +46,7 @@ get_time_range_schema = {
"type": "object",
"properties": {
"timeRange": {"type": "string"},
"shift": {"type": "string"},
},
},
}
@ -110,12 +111,16 @@ class Api(BaseSupersetView):
rv = []
for time_range in time_ranges:
since, until = get_since_until(time_range["timeRange"])
since, until = get_since_until(
time_range=time_range["timeRange"],
time_shift=time_range.get("shift"),
)
rv.append(
{
"since": since.isoformat() if since else "",
"until": until.isoformat() if until else "",
"timeRange": time_range["timeRange"],
"shift": time_range.get("shift"),
}
)
return self.json_response({"result": rv})

14
tests/integration_tests/charts/api_tests.py
View File

@ -1520,6 +1520,20 @@ class TestChartApi(ApiOwnersTestCaseMixin, InsertChartMixin, SupersetTestCase):
assert "until" in data["result"][0]
assert "timeRange" in data["result"][0]
humanize_time_range = [
{"timeRange": "2021-01-01 : 2022-02-01", "shift": "1 year ago"},
{"timeRange": "2022-01-01 : 2023-02-01", "shift": "2 year ago"},
]
uri = f"api/v1/time_range/?q={prison.dumps(humanize_time_range)}"
rv = self.client.get(uri)
data = json.loads(rv.data.decode("utf-8"))
assert rv.status_code == 200
assert len(data["result"]) == 2
assert "since" in data["result"][0]
assert "until" in data["result"][0]
assert "timeRange" in data["result"][0]
assert "shift" in data["result"][0]
def test_query_form_data(self):
"""
Chart API: Test query form data

315
tests/integration_tests/migrations/f84fde59123a_update_charts_with_old_time_comparison__test.py Normal file
View File

@ -0,0 +1,315 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from copy import deepcopy
from importlib import import_module
from typing import Any
migrate_time_comparison_to_new_format = import_module(
"superset.migrations.versions."
"2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison",
)
downgrade_comparison_params = (
migrate_time_comparison_to_new_format.downgrade_comparison_params
)
upgrade_comparison_params = (
migrate_time_comparison_to_new_format.upgrade_comparison_params
)
params_v1_with_custom: dict[str, Any] = {
"datasource": "2__table",
"viz_type": "pop_kpi",
"metric": {
"expressionType": "SIMPLE",
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_o6rj1h6jty_3t6mrruogfv",
},
"adhoc_filters": [
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 1986",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8",
}
],
"row_limit": 10000,
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_comparison": "c",
"enable_time_comparison": True,
"adhoc_custom": [
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1981-01-01 : 1983-01-01",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
}
],
}
params_v1_other_than_custom: dict[str, Any] = {
"datasource": "2__table",
"viz_type": "pop_kpi",
"metric": {
"expressionType": "SIMPLE",
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_96s7b8iypsr_4wrlgm0i7il",
},
"adhoc_filters": [
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 2000",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_2sefqq1rwb7_lhqvw7ukc6",
}
],
"row_limit": 10000,
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_comparison": "r",
"enable_time_comparison": True,
"adhoc_custom": [
{
"clause": "WHERE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "No filter",
"expressionType": "SIMPLE",
}
],
}
params_v2_with_custom: dict[str, Any] = {
"datasource": "2__table",
"viz_type": "pop_kpi",
"metric": {
"expressionType": "SIMPLE",
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_o6rj1h6jty_3t6mrruogfv",
},
"adhoc_filters": [
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 1986",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8",
}
],
"row_limit": 10000,
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_compare": "custom",
"comparison_type": "values",
"start_date_offset": "1981-01-01",
}
params_v2_other_than_custom: dict[str, Any] = {
"datasource": "2__table",
"viz_type": "pop_kpi",
"metric": {
"expressionType": "SIMPLE",
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_96s7b8iypsr_4wrlgm0i7il",
},
"adhoc_filters": [
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 2000",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_2sefqq1rwb7_lhqvw7ukc6",
}
],
"row_limit": 10000,
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_compare": "inherit",
"comparison_type": "values",
}
def test_upgrade_chart_params_with_custom():
"""
ensure that the new time comparison params are added
"""
original_params = deepcopy(params_v1_with_custom)
upgraded_params = upgrade_comparison_params(original_params)
assert upgraded_params == params_v2_with_custom
def test_downgrade_chart_params_with_custom():
"""
ensure that the params downgrade operation produces an almost identical dict
as the original value
"""
original_params = deepcopy(params_v2_with_custom)
downgraded_params = downgrade_comparison_params(original_params)
# Ignore any property called filterOptionName simce that uses a random hash
for adhoc_custom in downgraded_params["adhoc_custom"]:
adhoc_custom.pop("filterOptionName", None)
assert downgraded_params == params_v1_with_custom
def test_upgrade_chart_params_other_than_custom():
"""
ensure that the new time comparison params are added
"""
original_params = deepcopy(params_v1_other_than_custom)
upgraded_params = upgrade_comparison_params(original_params)
assert upgraded_params == params_v2_other_than_custom
def test_downgrade_chart_params_other_than_custom():
"""
ensure that the params downgrade operation produces an almost identical dict
as the original value
"""
original_params = deepcopy(params_v2_other_than_custom)
downgraded_params = downgrade_comparison_params(original_params)
assert downgraded_params == params_v1_other_than_custom

21
tests/unit_tests/utils/date_parser_tests.py
View File

@ -189,6 +189,27 @@ def test_get_since_until() -> None:
expected = datetime(2000, 1, 1), datetime(2018, 1, 1)
assert result == expected
result = get_since_until(
time_range="2000-01-01T00:00:00 : 2018-01-01T00:00:00",
time_shift="1 year ago",
)
expected = datetime(1999, 1, 1), datetime(2017, 1, 1)
assert result == expected
result = get_since_until(
time_range="2000-01-01T00:00:00 : 2018-01-01T00:00:00",
time_shift="1 month ago",
)
expected = datetime(1999, 12, 1), datetime(2017, 12, 1)
assert result == expected
result = get_since_until(
time_range="2000-01-01T00:00:00 : 2018-01-01T00:00:00",
time_shift="1 week ago",
)
expected = datetime(1999, 12, 25), datetime(2017, 12, 25)
assert result == expected
with pytest.raises(ValueError):
get_since_until(time_range="tomorrow : yesterday")