chore: Remove legacy SIP-15 interim logic/flags (#18936)

* chore: Remove legacy SIP-15 logic

* Update ab9a9d86e695_deprecate_time_range_endpoints.py

* Update UPDATING.md

* Update UPDATING.md

* Update UPDATING.md

Co-authored-by: John Bodley <john.bodley@airbnb.com>
This commit is contained in:
John Bodley 2022-03-04 09:15:36 +13:00 committed by GitHub
parent 6becd38e7f
commit 26486d01c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
111 changed files with 123 additions and 744 deletions

View File

@ -24,6 +24,8 @@ assists people when migrating to a new version.
## Next
- [18936](https://github.com/apache/superset/pull/18936): Removes legacy SIP-15 interm logic/flags—specifically the `SIP_15_ENABLED`, `SIP_15_GRACE_PERIOD_END`, `SIP_15_DEFAULT_TIME_RANGE_ENDPOINTS`, and `SIP_15_TOAST_MESSAGE` flags. Time range endpoints are no longer configurable and strictly adhere to the `[start, end)` paradigm, i.e., inclusive of the start and exclusive of the end. Additionally this change removes the now obsolete `time_range_endpoints` from the form-data and resulting in the cache being busted.
### Breaking Changes
- [18976](https://github.com/apache/superset/pull/18976): When running the app in debug mode, the app will default to use `SimpleCache` for `FILTER_STATE_CACHE_CONFIG` and `EXPLORE_FORM_DATA_CACHE_CONFIG`. When running in non-debug mode, a cache backend will need to be defined, otherwise the application will fail to start. For installations using Redis or other caching backends, it is recommended to use the same backend for both cache configs.

View File

@ -242,64 +242,3 @@ FEATURE_FLAGS = {
```
A current list of feature flags can be found in [RESOURCES/FEATURE_FLAGS.md](https://github.com/apache/superset/blob/master/RESOURCES/FEATURE_FLAGS.md).
### SIP 15
[Superset Improvement Proposal 15](https://github.com/apache/superset/issues/6360) aims to
ensure that time intervals are handled in a consistent and transparent manner for both the Druid and
SQLAlchemy connectors.
Prior to SIP-15 SQLAlchemy used inclusive endpoints however these may behave like exclusive for
string columns (due to lexicographical ordering) if no formatting was defined and the column
formatting did not conform to an ISO 8601 date-time (refer to the SIP for details).
To remedy this rather than having to define the date/time format for every non-IS0 8601 date-time
column, once can define a default column mapping on a per database level via the `extra` parameter:
```
{
"python_date_format_by_column_name": {
"ds": "%Y-%m-%d"
}
}
```
**New Deployments**
All new deployments should enable SIP-15 by setting this value in `superset_config.py`:
```
SIP_15_ENABLED = True
```
**Existing Deployments**
Given that it is not apparent whether the chart creator was aware of the time range inconsistencies
(and adjusted the endpoints accordingly) changing the behavior of all charts is overly aggressive.
Instead SIP-15 proivides a soft transistion allowing producers (chart owners) to see the impact of
the proposed change and adjust their charts accordingly.
Prior to enabling SIP-15, existing deployments should communicate to their users the impact of the
change and define a grace period end date (exclusive of course) after which all charts will conform
to the [start, end) interval.
```python
from datetime import date
SIP_15_ENABLED = True
SIP_15_GRACE_PERIOD_END = date(<YYYY>, <MM>, <DD>)
```
To aid with transparency the current endpoint behavior is explicitly called out in the chart time
range (post SIP-15 this will be [start, end) for all connectors and databases). One can override the
defaults on a per database level via the `extra` parameter.
```python
{
"time_range_endpoints": ["inclusive", "inclusive"]
}
```
Note in a future release the interim SIP-15 logic will be removed (including the
`time_grain_endpoints` form-data field) via a code change and Alembic migration.

View File

@ -683,10 +683,6 @@
"nullable": true,
"type": "string"
},
"time_range_endpoints": {
"items": {},
"type": "array"
},
"where": {
"description": "WHERE clause to be added to queries using AND operator.",
"type": "string"

View File

@ -412,7 +412,6 @@ describe('Nativefilters Sanity test', () => {
viz_type: 'echarts_timeseries',
datasource: '3__table',
granularity_sqla: 'purpose__last_set',
time_range_endpoints: ['inclusive', 'exclusive'],
time_grain_sqla: 'P1D',
time_range: 'No filter',
metrics: ['count'],

View File

@ -91,15 +91,6 @@ export const datasourceAndVizType: ControlPanelSectionConfig = {
),
},
},
{
name: 'time_range_endpoints',
config: {
type: 'HiddenControl',
label: t('Time range endpoints'),
hidden: true,
description: t('Time range endpoints (SIP-15)'),
},
},
],
],
};

View File

@ -323,7 +323,6 @@ const time_range: SharedControlConfig<'DateFilterControl'> = {
),
mapStateToProps: ({ datasource, form_data }) => ({
datasource,
endpoints: form_data?.time_range_endpoints || null,
}),
};

View File

@ -27,13 +27,7 @@ import {
export const DTTM_ALIAS = '__timestamp';
export const EXTRA_FORM_DATA_OVERRIDE_EXTRA_KEYS: (keyof ExtraFormDataOverrideExtras)[] =
[
'druid_time_origin',
'relative_start',
'relative_end',
'time_grain_sqla',
'time_range_endpoints',
];
['druid_time_origin', 'relative_start', 'relative_end', 'time_grain_sqla'];
export const EXTRA_FORM_DATA_APPEND_KEYS: (keyof ExtraFormDataAppend)[] = [
'adhoc_filters',

View File

@ -89,9 +89,5 @@ export default function extractExtras(formData: QueryFormData): ExtractedExtra {
delete extract.time_grain_sqla;
}
// map time range endpoints:
if (formData.time_range_endpoints)
extras.time_range_endpoints = formData.time_range_endpoints;
return extract;
}

View File

@ -19,7 +19,7 @@
*/
import { DatasourceType } from './Datasource';
import { BinaryOperator, SetOperator, UnaryOperator } from './Operator';
import { AppliedTimeExtras, TimeRange, TimeRangeEndpoints } from './Time';
import { AppliedTimeExtras, TimeRange } from './Time';
import { AnnotationLayer } from './AnnotationLayer';
import {
QueryFields,
@ -59,7 +59,6 @@ export type QueryObjectExtras = Partial<{
relative_start?: string;
relative_end?: string;
time_grain_sqla?: TimeGranularity;
time_range_endpoints?: TimeRangeEndpoints;
/** WHERE condition */
where?: string;
}>;

View File

@ -30,7 +30,7 @@ import {
QueryObjectExtras,
QueryObjectFilterClause,
} from './Query';
import { TimeRange, TimeRangeEndpoints } from './Time';
import { TimeRange } from './Time';
import { TimeGranularity } from '../../time-format';
import { JsonObject } from '../../connection';
import { AdhocColumn, PhysicalColumn } from './Column';
@ -120,11 +120,7 @@ export type ExtraFormDataAppend = {
* filter clauses can't be overridden */
export type ExtraFormDataOverrideExtras = Pick<
QueryObjectExtras,
| 'druid_time_origin'
| 'relative_start'
| 'relative_end'
| 'time_grain_sqla'
| 'time_range_endpoints'
'druid_time_origin' | 'relative_start' | 'relative_end' | 'time_grain_sqla'
>;
/** These parameters override those already present in the form data/query object */
@ -180,7 +176,6 @@ export interface BaseFormData extends TimeRange, FormDataResidual {
force?: boolean;
result_format?: string;
result_type?: string;
time_range_endpoints?: TimeRangeEndpoints;
annotation_layers?: AnnotationLayer[];
url_params?: Record<string, string>;
custom_params?: Record<string, string>;

View File

@ -37,7 +37,4 @@ export type AppliedTimeExtras = Partial<
Record<TimeColumnConfigKey, keyof QueryObject>
>;
export type TimeRangeEndpoint = 'unknown' | 'inclusive' | 'exclusive';
export type TimeRangeEndpoints = [TimeRangeEndpoint, TimeRangeEndpoint];
export default {};

View File

@ -30,7 +30,6 @@ describe('extractExtras', () => {
expect(
extractExtras({
...baseQueryFormData,
time_range_endpoints: ['inclusive', 'exclusive'],
extra_filters: [
{
col: '__time_col',
@ -57,7 +56,6 @@ describe('extractExtras', () => {
},
extras: {
time_grain_sqla: 'PT5M',
time_range_endpoints: ['inclusive', 'exclusive'],
},
filters: [],
granularity: 'ds2',
@ -107,7 +105,6 @@ describe('extractExtras', () => {
expect(
extractExtras({
...baseQueryFormData,
time_range_endpoints: ['inclusive', 'exclusive'],
extra_filters: [
{
col: 'gender',
@ -139,7 +136,6 @@ describe('extractExtras', () => {
},
extras: {
time_grain_sqla: 'PT5M',
time_range_endpoints: ['inclusive', 'exclusive'],
},
filters: [
{

View File

@ -26,7 +26,6 @@ export default {
datasource: '93829__table',
viz_type: 'deck_polygon',
url_params: {},
time_range_endpoints: ['inclusive', 'exclusive'],
granularity_sqla: null,
time_range: '100 years ago : ',
line_column: 'geometry',

View File

@ -129,8 +129,6 @@ const createProps = () => ({
row_limit: 10000,
show_legend: false,
time_range: 'No filter',
time_range_endpoints: ['inclusive', 'exclusive'],
url_params: {},
viz_type: 'dist_bar',
x_ticks_layout: 'auto',
y_axis_format: 'SMART_NUMBER',

View File

@ -66,7 +66,6 @@ const createProps = (viz_type = 'sunburst') => ({
row_limit: 10000,
slice_id: 371,
time_range: 'No filter',
time_range_endpoints: ['inclusive', 'exclusive'],
url_params: {},
viz_type,
},

View File

@ -967,7 +967,6 @@ const FiltersConfigForm = (
>
<DateFilterControl
name="time_range"
endpoints={['inclusive', 'exclusive']}
onChange={timeRange => {
setNativeFilterFieldValues(form, filterId, {
time_range: timeRange,

View File

@ -82,7 +82,6 @@ export const getFormData = ({
showSearch: true,
defaultValue: defaultDataMask?.filterState?.value,
time_range,
time_range_endpoints: ['inclusive', 'exclusive'],
url_params: extractUrlParams('regular'),
inView: true,
viz_type: filterType,

View File

@ -57,7 +57,6 @@ const regionFilter = {
show_bubbles: true,
slice_id: 32,
time_range: '2014-01-01 : 2014-01-02',
time_range_endpoints: ['inclusive', 'exclusive'],
viz_type: 'filter_box',
},
modified: '<bound method AuditMixinNullable.modified of Region Filter>',
@ -85,7 +84,6 @@ const chart1 = {
show_bubbles: true,
slice_id: 33,
time_range: '2000 : 2014-01-02',
time_range_endpoints: ['inclusive', 'exclusive'],
viz_type: 'big_number',
},
modified: "<bound method AuditMixinNullable.modified of World's Population>",

View File

@ -30,7 +30,6 @@ const createProps = () => ({
datasource: '34__table',
slice_id: 456,
url_params: {},
time_range_endpoints: ['unknown', 'inclusive'],
time_range: 'Last week',
all_columns_x: 'source',
all_columns_y: 'target',

View File

@ -31,7 +31,6 @@ const createProps = () => ({
datasource: '49__table',
slice_id: 318,
url_params: {},
time_range_endpoints: ['inclusive', 'exclusive'],
granularity_sqla: 'time_start',
time_range: 'No filter',
all_columns_x: ['age'],
@ -65,7 +64,6 @@ const createProps = () => ({
row_limit: 10000,
slice_id: 318,
time_range: 'No filter',
time_range_endpoints: ['inclusive', 'exclusive'],
url_params: {},
viz_type: 'histogram',
x_axis_label: 'age',

View File

@ -30,7 +30,6 @@ const createProps = () => ({
datasource: '49__table',
slice_id: 318,
url_params: {},
time_range_endpoints: ['inclusive', 'exclusive'],
granularity_sqla: 'time_start',
time_range: 'No filter',
all_columns_x: ['age'],
@ -66,7 +65,6 @@ const createProps = () => ({
row_limit: 10000,
slice_id: 318,
time_range: 'No filter',
time_range_endpoints: ['inclusive', 'exclusive'],
url_params: {},
viz_type: 'histogram',
x_axis_label: 'age',

View File

@ -48,7 +48,6 @@ const createProps = () => ({
row_limit: 10000,
slice_id: 318,
time_range: 'No filter',
time_range_endpoints: ['inclusive', 'exclusive'],
url_params: {},
viz_type: 'histogram',
x_axis_label: 'age',
@ -108,7 +107,7 @@ fetchMock.get('glob:*/api/v1/chart/318', {
},
],
params:
'{"adhoc_filters": [], "all_columns_x": ["age"], "color_scheme": "supersetColors", "datasource": "42__table", "granularity_sqla": "time_start", "groupby": null, "label_colors": {}, "link_length": "25", "queryFields": {"groupby": "groupby"}, "row_limit": 10000, "slice_id": 1380, "time_range": "No filter", "time_range_endpoints": ["inclusive", "exclusive"], "url_params": {}, "viz_type": "histogram", "x_axis_label": "age", "y_axis_label": "count"}',
'{"adhoc_filters": [], "all_columns_x": ["age"], "color_scheme": "supersetColors", "datasource": "42__table", "granularity_sqla": "time_start", "groupby": null, "label_colors": {}, "link_length": "25", "queryFields": {"groupby": "groupby"}, "row_limit": 10000, "slice_id": 1380, "time_range": "No filter", "url_params": {}, "viz_type": "histogram", "x_axis_label": "age", "y_axis_label": "count"}',
slice_name: 'Age distribution of respondents',
viz_type: 'histogram',
},

View File

@ -18,13 +18,7 @@
*/
import React, { useState, useEffect, useMemo } from 'react';
import rison from 'rison';
import {
SupersetClient,
styled,
t,
TimeRangeEndpoints,
useTheme,
} from '@superset-ui/core';
import { SupersetClient, styled, t, useTheme } from '@superset-ui/core';
import {
buildTimeRangeString,
formatTimeRange,
@ -71,10 +65,7 @@ const guessFrame = (timeRange: string): FrameType => {
return 'Advanced';
};
const fetchTimeRange = async (
timeRange: string,
endpoints?: TimeRangeEndpoints,
) => {
const fetchTimeRange = async (timeRange: string) => {
const query = rison.encode_uri(timeRange);
const endpoint = `/api/v1/time_range/?q=${query}`;
try {
@ -84,7 +75,7 @@ const fetchTimeRange = async (
response?.json?.result?.until || '',
);
return {
value: formatTimeRange(timeRangeString, endpoints),
value: formatTimeRange(timeRangeString),
};
} catch (response) {
const clientError = await getClientErrorObject(response);
@ -171,7 +162,6 @@ interface DateFilterControlProps {
name: string;
onChange: (timeRange: string) => void;
value?: string;
endpoints?: TimeRangeEndpoints;
type?: Type;
}
@ -181,7 +171,7 @@ export const getDateFilterControlTestId = testWithId(
);
export default function DateFilterLabel(props: DateFilterControlProps) {
const { value = DEFAULT_TIME_RANGE, endpoints, onChange, type } = props;
const { value = DEFAULT_TIME_RANGE, onChange, type } = props;
const [actualTimeRange, setActualTimeRange] = useState<string>(value);
const [show, setShow] = useState<boolean>(false);
@ -194,7 +184,7 @@ export default function DateFilterLabel(props: DateFilterControlProps) {
const [tooltipTitle, setTooltipTitle] = useState<string>(value);
useEffect(() => {
fetchTimeRange(value, endpoints).then(({ value: actualRange, error }) => {
fetchTimeRange(value).then(({ value: actualRange, error }) => {
if (error) {
setEvalResponse(error || '');
setValidTimeRange(false);
@ -235,18 +225,16 @@ export default function DateFilterLabel(props: DateFilterControlProps) {
useDebouncedEffect(
() => {
if (lastFetchedTimeRange !== timeRangeValue) {
fetchTimeRange(timeRangeValue, endpoints).then(
({ value: actualRange, error }) => {
if (error) {
setEvalResponse(error || '');
setValidTimeRange(false);
} else {
setEvalResponse(actualRange || '');
setValidTimeRange(true);
}
setLastFetchedTimeRange(timeRangeValue);
},
);
fetchTimeRange(timeRangeValue).then(({ value: actualRange, error }) => {
if (error) {
setEvalResponse(error || '');
setValidTimeRange(false);
} else {
setEvalResponse(actualRange || '');
setValidTimeRange(true);
}
setLastFetchedTimeRange(timeRangeValue);
});
}
},
SLOW_DEBOUNCE,

View File

@ -16,8 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
import { TimeRangeEndpoints } from '@superset-ui/core';
export const SEPARATOR = ' : ';
export const buildTimeRangeString = (since: string, until: string): string =>
@ -26,17 +24,11 @@ export const buildTimeRangeString = (since: string, until: string): string =>
const formatDateEndpoint = (dttm: string, isStart?: boolean): string =>
dttm.replace('T00:00:00', '') || (isStart ? '-∞' : '∞');
export const formatTimeRange = (
timeRange: string,
endpoints?: TimeRangeEndpoints,
) => {
export const formatTimeRange = (timeRange: string) => {
const splitDateRange = timeRange.split(SEPARATOR);
if (splitDateRange.length === 1) return timeRange;
const formattedEndpoints = (endpoints || ['unknown', 'unknown']).map(
(endpoint: string) => (endpoint === 'inclusive' ? '≤' : '<'),
);
return `${formatDateEndpoint(splitDateRange[0], true)} ${
formattedEndpoints[0]
} col ${formattedEndpoints[1]} ${formatDateEndpoint(splitDateRange[1])}`;
return `${formatDateEndpoint(
splitDateRange[0],
true,
)} col < ${formatDateEndpoint(splitDateRange[1])}`;
};

View File

@ -316,19 +316,16 @@ describe('formatTimeRange', () => {
expect(formatTimeRange('Last 7 days')).toBe('Last 7 days');
expect(formatTimeRange('No filter')).toBe('No filter');
expect(formatTimeRange('Yesterday : Tomorrow')).toBe(
'Yesterday < col < Tomorrow',
'Yesterday ≤ col < Tomorrow',
);
expect(formatTimeRange('2010-07-30T00:00:00 : 2020-07-30T00:00:00')).toBe(
'2010-07-30 ≤ col < 2020-07-30',
);
expect(formatTimeRange('2010-07-30T01:00:00 : ')).toBe(
'2010-07-30T01:00:00 ≤ col < ∞',
);
expect(
formatTimeRange('2010-07-30T00:00:00 : 2020-07-30T00:00:00', [
'inclusive',
'exclusive',
]),
).toBe('2010-07-30 ≤ col < 2020-07-30');
expect(
formatTimeRange('2010-07-30T01:00:00 : ', ['exclusive', 'inclusive']),
).toBe('2010-07-30T01:00:00 < col ≤ ∞');
expect(formatTimeRange(' : 2020-07-30T00:00:00')).toBe(
'-∞ < col < 2020-07-30',
'-∞ ≤ col < 2020-07-30',
);
});
});

View File

@ -62,15 +62,6 @@ export const datasourceAndVizType: ControlPanelSectionConfig = {
description: t('Extra parameters for use in jinja templated queries'),
},
},
{
name: 'time_range_endpoints',
config: {
type: 'HiddenControl',
label: t('Time range endpoints'),
hidden: true,
description: t('Time range endpoints (SIP-15)'),
},
},
],
],
};

View File

@ -348,10 +348,6 @@ export const controls = {
"using the engine's local timezone. Note one can explicitly set the timezone " +
'per the ISO 8601 format if specifying either the start and/or end time.',
),
mapStateToProps: ({ form_data: formData }) => ({
// eslint-disable-next-line camelcase
endpoints: formData?.time_range_endpoints,
}),
},
row_limit: {

View File

@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
import { styled, TimeRangeEndpoint } from '@superset-ui/core';
import { styled } from '@superset-ui/core';
import React, { useCallback, useEffect } from 'react';
import DateFilterControl from 'src/explore/components/controls/DateFilterControl';
import { NO_TIME_RANGE } from 'src/explore/constants';
@ -55,11 +55,6 @@ const ControlContainer = styled.div<{
}
`;
const endpoints = ['inclusive', 'exclusive'] as [
TimeRangeEndpoint,
TimeRangeEndpoint,
];
export default function TimeFilterPlugin(props: PluginFilterTimeProps) {
const {
setDataMask,
@ -105,7 +100,6 @@ export default function TimeFilterPlugin(props: PluginFilterTimeProps) {
onMouseLeave={unsetFocusedFilter}
>
<DateFilterControl
endpoints={endpoints}
value={filterState.value || NO_TIME_RANGE}
name="time_range"
onChange={handleTimeRangeChange}

View File

@ -33,7 +33,6 @@ from superset.utils.core import (
FilterOperator,
PostProcessingBoxplotWhiskerType,
PostProcessingContributionOrientation,
TimeRangeEndpoint,
)
if TYPE_CHECKING:
@ -822,7 +821,6 @@ class ChartDataFilterSchema(Schema):
class ChartDataExtrasSchema(Schema):
time_range_endpoints = fields.List(EnumField(TimeRangeEndpoint, by_value=True))
relative_start = fields.String(
description="Start time for relative time deltas. "
'Default: `config["DEFAULT_RELATIVE_START_TIME"]`',

View File

@ -16,12 +16,12 @@
# under the License.
from __future__ import annotations
from datetime import date, datetime
from datetime import datetime
from typing import Any, Dict, Optional, Tuple, TYPE_CHECKING
from superset.common.chart_data import ChartDataResultType
from superset.common.query_object import QueryObject
from superset.utils.core import apply_max_row_limit, DatasourceDict, TimeRangeEndpoint
from superset.utils.core import apply_max_row_limit, DatasourceDict
from superset.utils.date_parser import get_since_until
if TYPE_CHECKING:
@ -79,12 +79,10 @@ class QueryObjectFactory: # pylint: disable=too-few-public-methods
str(datasource["type"]), int(datasource["id"]), self._session_maker()
)
def _process_extras(self, extras: Optional[Dict[str, Any]]) -> Dict[str, Any]:
def _process_extras( # pylint: disable=no-self-use
self, extras: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
extras = extras or {}
if self._config["SIP_15_ENABLED"]:
extras["time_range_endpoints"] = self._determine_time_range_endpoints(
extras.get("time_range_endpoints")
)
return extras
def _process_row_limit(
@ -117,18 +115,3 @@ class QueryObjectFactory: # pylint: disable=too-few-public-methods
# light version of the view.utils.core
# import view.utils require application context
# Todo: move it and the view.utils.core to utils package
def _determine_time_range_endpoints(
self, raw_endpoints: Optional[Tuple[str, str]] = None,
) -> Optional[Tuple[TimeRangeEndpoint, TimeRangeEndpoint]]:
if (
self._config["SIP_15_GRACE_PERIOD_END"]
and date.today() >= self._config["SIP_15_GRACE_PERIOD_END"]
):
return TimeRangeEndpoint.INCLUSIVE, TimeRangeEndpoint.EXCLUSIVE
if raw_endpoints:
start, end = raw_endpoints
return TimeRangeEndpoint(start), TimeRangeEndpoint(end)
return TimeRangeEndpoint.INCLUSIVE, TimeRangeEndpoint.EXCLUSIVE

View File

@ -29,7 +29,7 @@ import os
import re
import sys
from collections import OrderedDict
from datetime import date, timedelta
from datetime import timedelta
from typing import Any, Callable, Dict, List, Optional, Type, TYPE_CHECKING, Union
import pkg_resources
@ -1258,22 +1258,6 @@ PREVENT_UNSAFE_DB_CONNECTIONS = True
# Example: SSL_CERT_PATH = "/certs"
SSL_CERT_PATH: Optional[str] = None
# SIP-15 should be enabled for all new Superset deployments which ensures that the time
# range endpoints adhere to [start, end). For existing deployments admins should provide
# a dedicated period of time to allow chart producers to update their charts before
# mass migrating all charts to use the [start, end) interval.
#
# Note if no end date for the grace period is specified then the grace period is
# indefinite.
SIP_15_ENABLED = True
SIP_15_GRACE_PERIOD_END: Optional[date] = None # exclusive
SIP_15_DEFAULT_TIME_RANGE_ENDPOINTS = ["unknown", "inclusive"]
SIP_15_TOAST_MESSAGE = (
"Action Required: Preview then save your chart using the "
'new time range endpoints <a target="_blank" href="{url}" '
'class="alert-link">here</a>.'
)
# Turn this key to False to disable ownership check on the old dataset MVC and
# datasource API /datasource/save.
#

View File

@ -301,35 +301,14 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
return self.table
def get_time_filter(
self,
start_dttm: DateTime,
end_dttm: DateTime,
time_range_endpoints: Optional[
Tuple[utils.TimeRangeEndpoint, utils.TimeRangeEndpoint]
],
self, start_dttm: DateTime, end_dttm: DateTime,
) -> ColumnElement:
col = self.get_sqla_col(label="__time")
l = []
if start_dttm:
l.append(
col
>= self.table.text(
self.dttm_sql_literal(start_dttm, time_range_endpoints)
)
)
l.append(col >= self.table.text(self.dttm_sql_literal(start_dttm)))
if end_dttm:
if (
time_range_endpoints
and time_range_endpoints[1] == utils.TimeRangeEndpoint.EXCLUSIVE
):
l.append(
col
< self.table.text(
self.dttm_sql_literal(end_dttm, time_range_endpoints)
)
)
else:
l.append(col <= self.table.text(self.dttm_sql_literal(end_dttm, None)))
l.append(col <= self.table.text(self.dttm_sql_literal(end_dttm)))
return and_(*l)
def get_timestamp_expression(
@ -368,13 +347,7 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
)
return self.table.make_sqla_column_compatible(time_expr, label)
def dttm_sql_literal(
self,
dttm: DateTime,
time_range_endpoints: Optional[
Tuple[utils.TimeRangeEndpoint, utils.TimeRangeEndpoint]
],
) -> str:
def dttm_sql_literal(self, dttm: DateTime) -> str:
"""Convert datetime object to a SQL expression string"""
sql = (
self.db_engine_spec.convert_dttm(self.type, dttm, db_extra=self.db_extra)
@ -387,12 +360,8 @@ class TableColumn(Model, BaseColumn, CertificationMixin):
tf = self.python_date_format
# Fallback to the default format (if defined) only if the SIP-15 time range
# endpoints, i.e., [start, end) are enabled.
if not tf and time_range_endpoints == (
utils.TimeRangeEndpoint.INCLUSIVE,
utils.TimeRangeEndpoint.EXCLUSIVE,
):
# Fallback to the default format (if defined).
if not tf:
tf = self.db_extra.get("python_date_format_by_column_name", {}).get(
self.column_name
)
@ -1210,8 +1179,6 @@ class SqlaTable(Model, BaseDatasource): # pylint: disable=too-many-public-metho
)
metrics_exprs = []
time_range_endpoints = extras.get("time_range_endpoints")
if granularity:
if granularity not in columns_by_name or not dttm_col:
raise QueryObjectValidationError(
@ -1238,12 +1205,10 @@ class SqlaTable(Model, BaseDatasource): # pylint: disable=too-many-public-metho
):
time_filters.append(
columns_by_name[self.main_dttm_col].get_time_filter(
from_dttm, to_dttm, time_range_endpoints
from_dttm, to_dttm,
)
)
time_filters.append(
dttm_col.get_time_filter(from_dttm, to_dttm, time_range_endpoints)
)
time_filters.append(dttm_col.get_time_filter(from_dttm, to_dttm))
# Always remove duplicates by column name, as sometimes `metrics_exprs`
# can have the same name as a groupby column (e.g. when users use
@ -1450,9 +1415,7 @@ class SqlaTable(Model, BaseDatasource): # pylint: disable=too-many-public-metho
if dttm_col and not db_engine_spec.time_groupby_inline:
inner_time_filter = [
dttm_col.get_time_filter(
inner_from_dttm or from_dttm,
inner_to_dttm or to_dttm,
time_range_endpoints,
inner_from_dttm or from_dttm, inner_to_dttm or to_dttm,
)
]
subq = subq.where(and_(*(where_clause_and + inner_time_filter)))

View File

@ -146,7 +146,6 @@ EXTRA_FORM_DATA_OVERRIDE_REGULAR_MAPPINGS = {
"time_range": "time_range",
"druid_time_origin": "druid_time_origin",
"time_grain_sqla": "time_grain_sqla",
"time_range_endpoints": "time_range_endpoints",
}
EXTRA_FORM_DATA_OVERRIDE_EXTRA_KEYS = {

View File

@ -174,7 +174,6 @@ def create_slices(tbl: SqlaTable, admin_owner: bool) -> Tuple[List[Slice], List[
"compare_suffix": "o10Y",
"limit": "25",
"time_range": "No filter",
"time_range_endpoints": ["inclusive", "exclusive"],
"granularity_sqla": "ds",
"groupby": [],
"row_limit": app.config["ROW_LIMIT"],

View File

@ -31,9 +31,6 @@ params:
row_limit: 10000
slice_id: 1380
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: histogram
x_axis_label: age

View File

@ -39,9 +39,6 @@ params:
show_legend: false
slice_id: 1383
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -39,9 +39,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -50,9 +50,6 @@ params:
metrics: metrics
row_limit: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
treemap_ratio: 1.618033988749895
url_params: {}
viz_type: treemap

View File

@ -57,9 +57,6 @@ params:
show_bubbles: true
slice_id: 1388
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: world_map
cache_timeout: null

View File

@ -49,9 +49,6 @@ params:
metric: metrics
row_limit: 1000
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
viz_type: chord
y_axis_format: SMART_NUMBER
cache_timeout: null

View File

@ -53,9 +53,6 @@ params:
sort_x_axis: alpha_asc
sort_y_axis: alpha_asc
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: heatmap
xscale_interval: null

View File

@ -54,9 +54,6 @@ params:
slice_id: 1361
subheader_font_size: 0.15
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: big_number_total
y_axis_format: SMART_NUMBER

View File

@ -69,9 +69,6 @@ params:
slice_id: 1376
time_grain_sqla: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: box_plot
whiskerOptions: Tukey

View File

@ -49,9 +49,6 @@ params:
metric: metrics
row_limit: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: sankey
cache_timeout: null

View File

@ -60,9 +60,6 @@ params:
queryFields: {}
slice_id: 1387
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: filter_box
cache_timeout: null

View File

@ -45,9 +45,6 @@ params:
queryFields: {}
slice_id: 3965
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: filter_box
cache_timeout: null

View File

@ -47,9 +47,6 @@ params:
show_legend: false
slice_id: 3632
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -58,9 +58,6 @@ params:
metric: metrics
row_limit: 10000
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: sankey
cache_timeout: null

View File

@ -67,9 +67,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -84,9 +84,6 @@ params:
metrics: metrics
row_limit: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
treemap_ratio: 1.618033988749895
url_params:
preselect_filters: '{"1389": {"platform": ["PS", "PS2", "PS3", "PS4"], "genre":

View File

@ -97,9 +97,6 @@ params:
slice_id: 3544
time_grain_sqla: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params:
preselect_filters: '{"1389": {"platform": ["PS", "PS2", "PS3", "PS4"], "genre":
null, "__time_range": "No filter"}}'

View File

@ -39,9 +39,6 @@ params:
show_legend: false
slice_id: 1385
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -66,9 +66,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -64,9 +64,6 @@ params:
sort_x_axis: alpha_asc
sort_y_axis: alpha_asc
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: heatmap
xscale_interval: null

View File

@ -87,9 +87,6 @@ params:
row_limit: null
slice_id: 1366
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: histogram
cache_timeout: null

View File

@ -48,9 +48,6 @@ params:
groupby: groupby
row_limit: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: histogram
cache_timeout: null

View File

@ -66,9 +66,6 @@ params:
show_bubbles: true
slice_id: 3635
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: world_map
cache_timeout: null

View File

@ -32,9 +32,6 @@ params:
row_limit: null
slice_id: 2396
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
treemap_ratio: 1.618033988749895
url_params: {}
viz_type: treemap

View File

@ -88,9 +88,6 @@ params:
stacked_style: stream
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: area
x_axis_format: smart_date

View File

@ -67,9 +67,6 @@ params:
show_legend: false
slice_id: 658
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -57,9 +57,6 @@ params:
subheader_font_size: 0.15
time_grain_sqla: P1M
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: big_number
y_axis_format: SMART_NUMBER

View File

@ -36,9 +36,6 @@ params:
slice_id: 1363
subheader_font_size: 0.15
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: big_number_total
y_axis_format: SMART_NUMBER

View File

@ -39,9 +39,6 @@ params:
sort_x_axis: alpha_asc
sort_y_axis: alpha_asc
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: heatmap
xscale_interval: null

View File

@ -80,9 +80,6 @@ params:
row_limit: 10
slice_id: 661
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
treemap_ratio: 1.618033988749895
url_params: {}
viz_type: treemap

View File

@ -27,9 +27,6 @@ params:
subheader: Slack Members
subheader_font_size: 0.125
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: big_number_total
y_axis_format: SMART_NUMBER

View File

@ -59,9 +59,6 @@ params:
show_legend: false
slice_id: 670
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -37,9 +37,6 @@ params:
sort_x_axis: alpha_asc
sort_y_axis: alpha_asc
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: heatmap
xscale_interval: null

View File

@ -51,9 +51,6 @@ params:
row_limit: 10
slice_id: 1377
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
treemap_ratio: 1.618033988749895
url_params: {}
viz_type: treemap

View File

@ -61,9 +61,6 @@ params:
stacked_style: stack
time_grain_sqla: P1M
time_range: '2003-01-01T00:00:00 : 2005-06-01T00:00:00'
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: area
x_axis_format: smart_date

View File

@ -46,9 +46,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -70,9 +70,6 @@ params:
time_compare: null
time_grain_sqla: P3M
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: bar
x_axis_format: '%m/%d/%Y'

View File

@ -73,9 +73,6 @@ params:
time_compare: null
time_grain_sqla: P3M
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: bar
x_axis_format: '%m/%d/%Y'

View File

@ -47,9 +47,6 @@ params:
show_legend: false
slice_id: 1365
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -62,9 +62,6 @@ params:
show_legend: true
time_grain_sqla: P1M
time_range: '2003-01-01T00:00:00 : 2005-06-01T00:00:00'
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: bar
x_axis_format: smart_date

View File

@ -114,9 +114,6 @@ params:
stacked_style: stream
time_grain_sqla: null
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params:
preselect_filters: '{"1389": {"platform": ["PS", "PS2", "PS3", "PS4"], "genre":
null, "__time_range": "No filter"}}'

View File

@ -51,9 +51,6 @@ params:
series_height: '25'
slice_id: 2811
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: horizon
cache_timeout: null

View File

@ -122,9 +122,6 @@ params:
show_legend: true
slice_id: 3546
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: dist_bar
x_ticks_layout: staggered

View File

@ -48,9 +48,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -42,9 +42,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -45,9 +45,6 @@ params:
subheader: ''
subheader_font_size: 0.15
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: big_number_total
y_axis_format: SMART_NUMBER

View File

@ -57,9 +57,6 @@ params:
table_timestamp_format: smart_date
time_grain_sqla: P1D
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: table
cache_timeout: null

View File

@ -46,9 +46,6 @@ params:
subheader: ''
subheader_font_size: 0.15
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: big_number_total
y_axis_format: $,.2f

View File

@ -168,9 +168,6 @@ params:
show_legend: true
slice_id: 3548
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params:
preselect_filters: '{"1389": {"platform": ["PS", "PS2", "PS3", "PS4"], "genre":
null, "__time_range": "No filter"}}'

View File

@ -37,9 +37,6 @@ params:
sort_x_axis: alpha_asc
sort_y_axis: alpha_asc
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: heatmap
xscale_interval: null

View File

@ -31,9 +31,6 @@ params:
metrics: metrics
row_limit: 10000
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
treemap_ratio: 1.618033988749895
url_params: {}
viz_type: treemap

View File

@ -51,9 +51,6 @@ params:
sqlExpression: null
show_bubbles: true
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: world_map
cache_timeout: null

View File

@ -32,9 +32,6 @@ params:
row_limit: 10000
slice_id: 3964
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: sunburst
cache_timeout: null

View File

@ -33,9 +33,6 @@ params:
sort_x_axis: alpha_asc
sort_y_axis: alpha_asc
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: heatmap
xscale_interval: null

View File

@ -38,9 +38,6 @@ params:
show_legend: false
slice_id: 3957
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -30,9 +30,6 @@ params:
row_limit: 10000
show_legend: false
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: dist_bar
x_ticks_layout: auto

View File

@ -39,9 +39,6 @@ params:
queryFields: {}
slice_id: 671
time_range: '2003-01-01T00:00:00 : 2005-06-01T00:00:00'
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: filter_box
cache_timeout: null

View File

@ -45,9 +45,6 @@ params:
granularity_sqla: Year
queryFields: {}
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params:
preselect_filters: '{"1389": {"platform": ["PS", "PS2", "PS3", "PS4"], "genre":
null, "__time_range": "No filter"}}'

View File

@ -38,9 +38,6 @@ params:
subheader_font_size: 0.15
time_grain_sqla: P1W
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
time_range_fixed: false
url_params: {}
viz_type: big_number

View File

@ -37,9 +37,6 @@ params:
subheader_font_size: 0.15
time_grain_sqla: P1W
time_range: '2020-08-05T00:00:00 : 2020-09-06T00:00:00'
time_range_endpoints:
- inclusive
- exclusive
time_range_fixed: false
url_params: {}
viz_type: big_number

View File

@ -47,9 +47,6 @@ params:
show_legend: false
slice_id: 1370
time_range: No filter
time_range_endpoints:
- inclusive
- exclusive
url_params: {}
viz_type: pie
cache_timeout: null

View File

@ -190,7 +190,6 @@ def load_deck_dash() -> None: # pylint: disable=too-many-statements
"max_radius": 250,
"row_limit": 5000,
"time_range": " : ",
"time_range_endpoints": ["inclusive", "exclusive"],
"size": "count",
"time_grain_sqla": None,
"viewport": {

View File

@ -161,7 +161,6 @@ def create_slices(tbl: BaseDatasource) -> List[Slice]:
"since": "2014-01-01",
"until": "2014-01-02",
"time_range": "2014-01-01 : 2014-01-02",
"time_range_endpoints": ["inclusive", "exclusive"],
"markup_type": "markdown",
"country_fieldtype": "cca3",
"entity": "country_code",

View File

@ -0,0 +1,59 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""deprecate time_range_endpoints
Revision ID: ab9a9d86e695
Revises: b5a422d8e252
Create Date: 2022-02-25 08:06:14.835094
"""
import json
from alembic import op
from sqlalchemy import Column, Integer, Text
from sqlalchemy.ext.declarative import declarative_base
from superset import db
# revision identifiers, used by Alembic.
revision = "ab9a9d86e695"
down_revision = "b5a422d8e252"
Base = declarative_base()
class Slice(Base):
__tablename__ = "slices"
id = Column(Integer, primary_key=True)
params = Column(Text)
def upgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
for slc in session.query(Slice):
params = json.loads(slc.params)
params.pop("time_range_endpoints", None)
slc.params = json.dumps(params)
session.commit()
session.close()
def downgrade():
pass

View File

@ -314,22 +314,6 @@ class RowLevelSecurityFilterType(str, Enum):
BASE = "Base"
class TimeRangeEndpoint(str, Enum):
"""
The time range endpoint types which represent inclusive, exclusive, or unknown.
Unknown represents endpoints which are ill-defined as though the interval may be
[start, end] the filter may behave like (start, end] due to mixed data types and
lexicographical ordering.
:see: https://github.com/apache/superset/issues/6360
"""
EXCLUSIVE = "exclusive"
INCLUSIVE = "inclusive"
UNKNOWN = "unknown"
class TemporalType(str, Enum):
"""
Supported temporal types

View File

@ -28,7 +28,7 @@ import backoff
import humanize
import pandas as pd
import simplejson as json
from flask import abort, flash, g, Markup, redirect, render_template, request, Response
from flask import abort, flash, g, redirect, render_template, request, Response
from flask_appbuilder import expose
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_appbuilder.security.decorators import (
@ -43,7 +43,6 @@ from sqlalchemy.engine.url import make_url
from sqlalchemy.exc import ArgumentError, DBAPIError, NoSuchModuleError, SQLAlchemyError
from sqlalchemy.orm.session import Session
from sqlalchemy.sql import functions as func
from werkzeug.urls import Href
from superset import (
app,
@ -769,35 +768,6 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
)
query_context = request.form.get("query_context")
# Flash the SIP-15 message if the slice is owned by the current user and has not
# been updated, i.e., is not using the [start, end) interval.
if (
config["SIP_15_ENABLED"]
and slc
and g.user in slc.owners
and (
not form_data.get("time_range_endpoints")
or form_data["time_range_endpoints"]
!= (
utils.TimeRangeEndpoint.INCLUSIVE,
utils.TimeRangeEndpoint.EXCLUSIVE,
)
)
):
url = Href("/superset/explore/")(
{
"form_data": json.dumps(
{
"slice_id": slc.id,
"time_range_endpoints": (
utils.TimeRangeEndpoint.INCLUSIVE.value,
utils.TimeRangeEndpoint.EXCLUSIVE.value,
),
}
)
}
)
flash(Markup(config["SIP_15_TOAST_MESSAGE"].format(url=url)))
try:
datasource_id, datasource_type = get_datasource_info(

Some files were not shown because too many files have changed in this diff Show More