mirror of https://github.com/apache/superset.git
[explore] improved filters (#2330)
* Support more filter operators * more filter operators [>, <, >=, <=, ==, !=, LIKE] * Fix need to escape/double `%` in LIKE clauses * spinner while loading values when changing column * datasource config elements to allow to applying predicates when fetching filter values * refactor * Removing doubling parens * rebasing * Merging migrations
This commit is contained in:
parent
82bc907088
commit
8042ac876e
1
setup.py
1
setup.py
|
@ -67,7 +67,6 @@ setup(
|
||||||
'sqlparse==0.1.19',
|
'sqlparse==0.1.19',
|
||||||
'thrift>=0.9.3',
|
'thrift>=0.9.3',
|
||||||
'thrift-sasl>=0.2.1',
|
'thrift-sasl>=0.2.1',
|
||||||
'werkzeug==0.11.15',
|
|
||||||
],
|
],
|
||||||
extras_require={
|
extras_require={
|
||||||
'cors': ['Flask-Cors>=2.0.0'],
|
'cors': ['Flask-Cors>=2.0.0'],
|
||||||
|
|
|
@ -4,8 +4,22 @@ import Select from 'react-select';
|
||||||
import { Button, Row, Col } from 'react-bootstrap';
|
import { Button, Row, Col } from 'react-bootstrap';
|
||||||
import SelectControl from './SelectControl';
|
import SelectControl from './SelectControl';
|
||||||
|
|
||||||
const arrayFilterOps = ['in', 'not in'];
|
const operatorsArr = [
|
||||||
const strFilterOps = ['==', '!=', '>', '<', '>=', '<=', 'regex'];
|
{ val: 'in', type: 'array', useSelect: true, multi: true },
|
||||||
|
{ val: 'not in', type: 'array', useSelect: true, multi: true },
|
||||||
|
{ val: '==', type: 'string', useSelect: true, multi: false },
|
||||||
|
{ val: '!=', type: 'string', useSelect: true, multi: false },
|
||||||
|
{ val: '>=', type: 'string' },
|
||||||
|
{ val: '<=', type: 'string' },
|
||||||
|
{ val: '>', type: 'string' },
|
||||||
|
{ val: '<', type: 'string' },
|
||||||
|
{ val: 'regex', type: 'string', datasourceTypes: ['druid'] },
|
||||||
|
{ val: 'LIKE', type: 'string', datasourceTypes: ['table'] },
|
||||||
|
];
|
||||||
|
const operators = {};
|
||||||
|
operatorsArr.forEach(op => {
|
||||||
|
operators[op.val] = op;
|
||||||
|
});
|
||||||
|
|
||||||
const propTypes = {
|
const propTypes = {
|
||||||
choices: PropTypes.array,
|
choices: PropTypes.array,
|
||||||
|
@ -13,11 +27,9 @@ const propTypes = {
|
||||||
removeFilter: PropTypes.func,
|
removeFilter: PropTypes.func,
|
||||||
filter: PropTypes.object.isRequired,
|
filter: PropTypes.object.isRequired,
|
||||||
datasource: PropTypes.object,
|
datasource: PropTypes.object,
|
||||||
having: PropTypes.bool,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const defaultProps = {
|
const defaultProps = {
|
||||||
having: false,
|
|
||||||
changeFilter: () => {},
|
changeFilter: () => {},
|
||||||
removeFilter: () => {},
|
removeFilter: () => {},
|
||||||
choices: [],
|
choices: [],
|
||||||
|
@ -27,102 +39,91 @@ const defaultProps = {
|
||||||
export default class Filter extends React.Component {
|
export default class Filter extends React.Component {
|
||||||
constructor(props) {
|
constructor(props) {
|
||||||
super(props);
|
super(props);
|
||||||
const filterOps = props.datasource.type === 'table' ?
|
this.state = {
|
||||||
['in', 'not in'] : ['==', '!=', 'in', 'not in', 'regex'];
|
valuesLoading: false,
|
||||||
this.opChoices = this.props.having ? ['==', '!=', '>', '<', '>=', '<=']
|
};
|
||||||
: filterOps;
|
}
|
||||||
|
componentDidMount() {
|
||||||
|
this.fetchFilterValues(this.props.filter.col);
|
||||||
}
|
}
|
||||||
fetchFilterValues(col) {
|
fetchFilterValues(col) {
|
||||||
if (!this.props.datasource) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const datasource = this.props.datasource;
|
const datasource = this.props.datasource;
|
||||||
let choices = [];
|
if (col && this.props.datasource && this.props.datasource.filter_select) {
|
||||||
if (col) {
|
this.setState({ valuesLoading: true });
|
||||||
$.ajax({
|
$.ajax({
|
||||||
type: 'GET',
|
type: 'GET',
|
||||||
url: `/superset/filter/${datasource.type}/${datasource.id}/${col}/`,
|
url: `/superset/filter/${datasource.type}/${datasource.id}/${col}/`,
|
||||||
success: (data) => {
|
success: (data) => {
|
||||||
choices = Object.keys(data).map((k) =>
|
this.props.changeFilter('choices', data);
|
||||||
([`'${data[k]}'`, `'${data[k]}'`]));
|
this.setState({ valuesLoading: false });
|
||||||
this.props.changeFilter('choices', choices);
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
switchFilterValue(prevFilter, nextOp) {
|
switchFilterValue(prevOp, nextOp) {
|
||||||
const prevOp = prevFilter.op;
|
if (operators[prevOp].type !== operators[nextOp].type) {
|
||||||
let newVal = null;
|
const val = this.props.filter.value;
|
||||||
if (arrayFilterOps.indexOf(prevOp) !== -1
|
let newVal;
|
||||||
&& strFilterOps.indexOf(nextOp) !== -1) {
|
|
||||||
// switch from array to string
|
// switch from array to string
|
||||||
newVal = this.props.filter.val.length > 0 ? this.props.filter.val[0] : '';
|
if (operators[nextOp].type === 'string' && val && val.length > 0) {
|
||||||
|
newVal = val[0];
|
||||||
|
} else if (operators[nextOp].type === 'string' && val) {
|
||||||
|
newVal = [val];
|
||||||
}
|
}
|
||||||
if (strFilterOps.indexOf(prevOp) !== -1
|
this.props.changeFilter('val', newVal);
|
||||||
&& arrayFilterOps.indexOf(nextOp) !== -1) {
|
|
||||||
// switch from string to array
|
|
||||||
newVal = this.props.filter.val === '' ? [] : [this.props.filter.val];
|
|
||||||
}
|
}
|
||||||
return newVal;
|
|
||||||
}
|
}
|
||||||
changeFilter(control, event) {
|
changeText(event) {
|
||||||
let value = event;
|
this.props.changeFilter('val', event.target.value);
|
||||||
if (event && event.target) {
|
|
||||||
value = event.target.value;
|
|
||||||
}
|
}
|
||||||
if (event && event.value) {
|
changeSelect(value) {
|
||||||
value = event.value;
|
this.props.changeFilter('val', value);
|
||||||
}
|
}
|
||||||
if (control === 'op') {
|
changeColumn(event) {
|
||||||
const newVal = this.switchFilterValue(this.props.filter, value);
|
this.props.changeFilter('col', event.value);
|
||||||
if (newVal) {
|
this.fetchFilterValues(event.value);
|
||||||
this.props.changeFilter(['op', 'val'], [value, newVal]);
|
|
||||||
} else {
|
|
||||||
this.props.changeFilter(control, value);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.props.changeFilter(control, value);
|
|
||||||
}
|
|
||||||
if (control === 'col' && value !== null && this.props.datasource.filter_select) {
|
|
||||||
this.fetchFilterValues(value);
|
|
||||||
}
|
}
|
||||||
|
changeOp(event) {
|
||||||
|
this.switchFilterValue(this.props.filter.op, event.value);
|
||||||
|
this.props.changeFilter('op', event.value);
|
||||||
}
|
}
|
||||||
removeFilter(filter) {
|
removeFilter(filter) {
|
||||||
this.props.removeFilter(filter);
|
this.props.removeFilter(filter);
|
||||||
}
|
}
|
||||||
renderFilterFormControl(filter) {
|
renderFilterFormControl(filter) {
|
||||||
const datasource = this.props.datasource;
|
const operator = operators[filter.op];
|
||||||
if (datasource && datasource.filter_select) {
|
if (operator.useSelect) {
|
||||||
if (!filter.choices) {
|
return (
|
||||||
this.fetchFilterValues(filter.col);
|
<SelectControl
|
||||||
|
multi={operator.multi}
|
||||||
|
freeForm
|
||||||
|
name="filter-value"
|
||||||
|
value={filter.val}
|
||||||
|
isLoading={this.state.valuesLoading}
|
||||||
|
choices={filter.choices}
|
||||||
|
onChange={this.changeSelect.bind(this)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// switching filter value between array/string when needed
|
|
||||||
if (strFilterOps.indexOf(filter.op) !== -1) {
|
|
||||||
// druid having filter or regex/==/!= filters
|
|
||||||
return (
|
return (
|
||||||
<input
|
<input
|
||||||
type="text"
|
type="text"
|
||||||
onChange={this.changeFilter.bind(this, 'val')}
|
onChange={this.changeText.bind(this)}
|
||||||
value={filter.val}
|
value={filter.val}
|
||||||
className="form-control input-sm"
|
className="form-control input-sm"
|
||||||
placeholder="Filter value"
|
placeholder="Filter value"
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return (
|
|
||||||
<SelectControl
|
|
||||||
multi
|
|
||||||
freeForm
|
|
||||||
name="filter-value"
|
|
||||||
value={filter.val}
|
|
||||||
choices={filter.choices || []}
|
|
||||||
onChange={this.changeFilter.bind(this, 'val')}
|
|
||||||
/>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
render() {
|
render() {
|
||||||
|
const datasource = this.props.datasource;
|
||||||
const filter = this.props.filter;
|
const filter = this.props.filter;
|
||||||
|
const opsChoices = operatorsArr
|
||||||
|
.filter(o => !o.datasourceTypes || o.datasourceTypes.indexOf(datasource.type) >= 0)
|
||||||
|
.map(o => ({ value: o.val, label: o.val }));
|
||||||
|
const colChoices = datasource ?
|
||||||
|
datasource.filterable_cols.map(c => ({ value: c[0], label: c[1] })) :
|
||||||
|
null;
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
<Row className="space-1">
|
<Row className="space-1">
|
||||||
|
@ -130,9 +131,10 @@ export default class Filter extends React.Component {
|
||||||
<Select
|
<Select
|
||||||
id="select-col"
|
id="select-col"
|
||||||
placeholder="Select column"
|
placeholder="Select column"
|
||||||
options={this.props.choices.map((c) => ({ value: c[0], label: c[1] }))}
|
clearable={false}
|
||||||
|
options={colChoices}
|
||||||
value={filter.col}
|
value={filter.col}
|
||||||
onChange={this.changeFilter.bind(this, 'col')}
|
onChange={this.changeColumn.bind(this)}
|
||||||
/>
|
/>
|
||||||
</Col>
|
</Col>
|
||||||
</Row>
|
</Row>
|
||||||
|
@ -141,9 +143,10 @@ export default class Filter extends React.Component {
|
||||||
<Select
|
<Select
|
||||||
id="select-op"
|
id="select-op"
|
||||||
placeholder="Select operator"
|
placeholder="Select operator"
|
||||||
options={this.opChoices.map((o) => ({ value: o, label: o }))}
|
options={opsChoices}
|
||||||
|
clearable={false}
|
||||||
value={filter.op}
|
value={filter.op}
|
||||||
onChange={this.changeFilter.bind(this, 'op')}
|
onChange={this.changeOp.bind(this)}
|
||||||
/>
|
/>
|
||||||
</Col>
|
</Col>
|
||||||
<Col md={7}>
|
<Col md={7}>
|
||||||
|
|
|
@ -4,14 +4,12 @@ import Filter from './Filter';
|
||||||
|
|
||||||
const propTypes = {
|
const propTypes = {
|
||||||
name: PropTypes.string,
|
name: PropTypes.string,
|
||||||
choices: PropTypes.array,
|
|
||||||
onChange: PropTypes.func,
|
onChange: PropTypes.func,
|
||||||
value: PropTypes.array,
|
value: PropTypes.array,
|
||||||
datasource: PropTypes.object,
|
datasource: PropTypes.object,
|
||||||
};
|
};
|
||||||
|
|
||||||
const defaultProps = {
|
const defaultProps = {
|
||||||
choices: [],
|
|
||||||
onChange: () => {},
|
onChange: () => {},
|
||||||
value: [],
|
value: [],
|
||||||
};
|
};
|
||||||
|
@ -19,8 +17,11 @@ const defaultProps = {
|
||||||
export default class FilterControl extends React.Component {
|
export default class FilterControl extends React.Component {
|
||||||
addFilter() {
|
addFilter() {
|
||||||
const newFilters = Object.assign([], this.props.value);
|
const newFilters = Object.assign([], this.props.value);
|
||||||
|
const col = this.props.datasource && this.props.datasource.filterable_cols.length > 0 ?
|
||||||
|
this.props.datasource.filterable_cols[0][0] :
|
||||||
|
null;
|
||||||
newFilters.push({
|
newFilters.push({
|
||||||
col: null,
|
col,
|
||||||
op: 'in',
|
op: 'in',
|
||||||
val: this.props.datasource.filter_select ? [] : '',
|
val: this.props.datasource.filter_select ? [] : '',
|
||||||
});
|
});
|
||||||
|
@ -43,22 +44,17 @@ export default class FilterControl extends React.Component {
|
||||||
this.props.onChange(this.props.value.filter((f, i) => i !== index));
|
this.props.onChange(this.props.value.filter((f, i) => i !== index));
|
||||||
}
|
}
|
||||||
render() {
|
render() {
|
||||||
const filters = [];
|
const filters = this.props.value.map((filter, i) => (
|
||||||
this.props.value.forEach((filter, i) => {
|
|
||||||
const filterBox = (
|
|
||||||
<div key={i}>
|
<div key={i}>
|
||||||
<Filter
|
<Filter
|
||||||
having={this.props.name === 'having_filters'}
|
having={this.props.name === 'having_filters'}
|
||||||
filter={filter}
|
filter={filter}
|
||||||
choices={this.props.choices}
|
|
||||||
datasource={this.props.datasource}
|
datasource={this.props.datasource}
|
||||||
removeFilter={this.removeFilter.bind(this, i)}
|
removeFilter={this.removeFilter.bind(this, i)}
|
||||||
changeFilter={this.changeFilter.bind(this, i)}
|
changeFilter={this.changeFilter.bind(this, i)}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
);
|
));
|
||||||
filters.push(filterBox);
|
|
||||||
});
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
{filters}
|
{filters}
|
||||||
|
|
|
@ -47,30 +47,39 @@ export default class SelectControl extends React.PureComponent {
|
||||||
this.props.onChange(optionValue);
|
this.props.onChange(optionValue);
|
||||||
}
|
}
|
||||||
getOptions(props) {
|
getOptions(props) {
|
||||||
const options = props.choices.map((c) => {
|
// Accepts different formats of input
|
||||||
|
const options = props.choices.map(c => {
|
||||||
|
let option;
|
||||||
|
if (Array.isArray(c)) {
|
||||||
const label = c.length > 1 ? c[1] : c[0];
|
const label = c.length > 1 ? c[1] : c[0];
|
||||||
const newOptions = {
|
option = {
|
||||||
value: c[0],
|
value: c[0],
|
||||||
label,
|
label,
|
||||||
};
|
};
|
||||||
if (c[2]) newOptions.imgSrc = c[2];
|
if (c[2]) option.imgSrc = c[2];
|
||||||
return newOptions;
|
} else if (Object.is(c)) {
|
||||||
|
option = c;
|
||||||
|
} else {
|
||||||
|
option = {
|
||||||
|
value: c,
|
||||||
|
label: c,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return option;
|
||||||
});
|
});
|
||||||
if (props.freeForm) {
|
if (props.freeForm) {
|
||||||
// For FreeFormSelect, insert value into options if not exist
|
// For FreeFormSelect, insert value into options if not exist
|
||||||
const values = props.choices.map((c) => c[0]);
|
const values = options.map(c => c.value);
|
||||||
if (props.value) {
|
if (props.value) {
|
||||||
if (typeof props.value === 'object') {
|
let valuesToAdd = props.value;
|
||||||
props.value.forEach((v) => {
|
if (!Array.isArray(valuesToAdd)) {
|
||||||
if (values.indexOf(v) === -1) {
|
valuesToAdd = [valuesToAdd];
|
||||||
|
}
|
||||||
|
valuesToAdd.forEach(v => {
|
||||||
|
if (values.indexOf(v) < 0) {
|
||||||
options.push({ value: v, label: v });
|
options.push({ value: v, label: v });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
} else {
|
|
||||||
if (values.indexOf(props.value) === -1) {
|
|
||||||
options.push({ value: props.value, label: props.value });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return options;
|
return options;
|
||||||
|
|
|
@ -1177,7 +1177,6 @@ export const controls = {
|
||||||
default: [],
|
default: [],
|
||||||
description: '',
|
description: '',
|
||||||
mapStateToProps: (state) => ({
|
mapStateToProps: (state) => ({
|
||||||
choices: (state.datasource) ? state.datasource.filterable_cols : [],
|
|
||||||
datasource: state.datasource,
|
datasource: state.datasource,
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
|
|
|
@ -17,6 +17,7 @@ const defaultProps = {
|
||||||
id: 1,
|
id: 1,
|
||||||
type: 'table',
|
type: 'table',
|
||||||
filter_select: false,
|
filter_select: false,
|
||||||
|
filterable_cols: ['country_name'],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -10,11 +10,8 @@ import Filter from '../../../../javascripts/explorev2/components/controls/Filter
|
||||||
import SelectControl from '../../../../javascripts/explorev2/components/controls/SelectControl';
|
import SelectControl from '../../../../javascripts/explorev2/components/controls/SelectControl';
|
||||||
|
|
||||||
const defaultProps = {
|
const defaultProps = {
|
||||||
choices: ['country_name'],
|
|
||||||
changeFilter: sinon.spy(),
|
changeFilter: sinon.spy(),
|
||||||
removeFilter: () => {
|
removeFilter: () => {},
|
||||||
// noop
|
|
||||||
},
|
|
||||||
filter: {
|
filter: {
|
||||||
col: null,
|
col: null,
|
||||||
op: 'in',
|
op: 'in',
|
||||||
|
@ -22,8 +19,9 @@ const defaultProps = {
|
||||||
},
|
},
|
||||||
datasource: {
|
datasource: {
|
||||||
id: 1,
|
id: 1,
|
||||||
type: 'table',
|
type: 'qtable',
|
||||||
filter_select: false,
|
filter_select: false,
|
||||||
|
filterable_cols: ['col1', 'col2'],
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -44,7 +42,7 @@ describe('Filter', () => {
|
||||||
expect(wrapper.find(Select)).to.have.lengthOf(2);
|
expect(wrapper.find(Select)).to.have.lengthOf(2);
|
||||||
expect(wrapper.find(Button)).to.have.lengthOf(1);
|
expect(wrapper.find(Button)).to.have.lengthOf(1);
|
||||||
expect(wrapper.find(SelectControl)).to.have.lengthOf(1);
|
expect(wrapper.find(SelectControl)).to.have.lengthOf(1);
|
||||||
expect(wrapper.find('#select-op').prop('options')).to.have.lengthOf(2);
|
expect(wrapper.find('#select-op').prop('options')).to.have.lengthOf(8);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders five op choices for table datasource', () => {
|
it('renders five op choices for table datasource', () => {
|
||||||
|
@ -53,16 +51,17 @@ describe('Filter', () => {
|
||||||
id: 1,
|
id: 1,
|
||||||
type: 'druid',
|
type: 'druid',
|
||||||
filter_select: false,
|
filter_select: false,
|
||||||
|
filterable_cols: ['country_name'],
|
||||||
};
|
};
|
||||||
const druidWrapper = shallow(<Filter {...props} />);
|
const druidWrapper = shallow(<Filter {...props} />);
|
||||||
expect(druidWrapper.find('#select-op').prop('options')).to.have.lengthOf(5);
|
expect(druidWrapper.find('#select-op').prop('options')).to.have.lengthOf(9);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders six op choices for having filter', () => {
|
it('renders six op choices for having filter', () => {
|
||||||
const props = defaultProps;
|
const props = defaultProps;
|
||||||
props.having = true;
|
props.having = true;
|
||||||
const havingWrapper = shallow(<Filter {...props} />);
|
const havingWrapper = shallow(<Filter {...props} />);
|
||||||
expect(havingWrapper.find('#select-op').prop('options')).to.have.lengthOf(6);
|
expect(havingWrapper.find('#select-op').prop('options')).to.have.lengthOf(9);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('calls changeFilter when select is changed', () => {
|
it('calls changeFilter when select is changed', () => {
|
||||||
|
|
|
@ -326,6 +326,7 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
is_hidden = Column(Boolean, default=False)
|
is_hidden = Column(Boolean, default=False)
|
||||||
filter_select_enabled = Column(Boolean, default=False)
|
filter_select_enabled = Column(Boolean, default=False)
|
||||||
description = Column(Text)
|
description = Column(Text)
|
||||||
|
fetch_values_from = Column(String(100))
|
||||||
default_endpoint = Column(Text)
|
default_endpoint = Column(Text)
|
||||||
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
||||||
owner = relationship(
|
owner = relationship(
|
||||||
|
@ -696,18 +697,18 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
|
|
||||||
def values_for_column(self,
|
def values_for_column(self,
|
||||||
column_name,
|
column_name,
|
||||||
from_dttm,
|
limit=10000):
|
||||||
to_dttm,
|
|
||||||
limit=500):
|
|
||||||
"""Retrieve some values for the given column"""
|
"""Retrieve some values for the given column"""
|
||||||
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
|
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
|
||||||
from_dttm = from_dttm.replace(tzinfo=DRUID_TZ)
|
if self.fetch_values_from:
|
||||||
to_dttm = to_dttm.replace(tzinfo=DRUID_TZ)
|
from_dttm = utils.parse_human_datetime(self.fetch_values_from)
|
||||||
|
else:
|
||||||
|
from_dttm = datetime(1970, 1, 1)
|
||||||
|
|
||||||
qry = dict(
|
qry = dict(
|
||||||
datasource=self.datasource_name,
|
datasource=self.datasource_name,
|
||||||
granularity="all",
|
granularity="all",
|
||||||
intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(),
|
intervals=from_dttm.isoformat() + '/' + datetime.now().isoformat(),
|
||||||
aggregations=dict(count=count("count")),
|
aggregations=dict(count=count("count")),
|
||||||
dimension=column_name,
|
dimension=column_name,
|
||||||
metric="count",
|
metric="count",
|
||||||
|
@ -718,10 +719,7 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
client.topn(**qry)
|
client.topn(**qry)
|
||||||
df = client.export_pandas()
|
df = client.export_pandas()
|
||||||
|
|
||||||
if df is None or df.size == 0:
|
return [row[0] for row in df.to_records(index=False)]
|
||||||
raise Exception(_("No data was returned."))
|
|
||||||
|
|
||||||
return df
|
|
||||||
|
|
||||||
def get_query_str( # noqa / druid
|
def get_query_str( # noqa / druid
|
||||||
self, client, qry_start_dttm,
|
self, client, qry_start_dttm,
|
||||||
|
@ -1015,6 +1013,14 @@ class DruidDatasource(Model, BaseDatasource):
|
||||||
cond = ~cond
|
cond = ~cond
|
||||||
elif op == 'regex':
|
elif op == 'regex':
|
||||||
cond = Filter(type="regex", pattern=eq, dimension=col)
|
cond = Filter(type="regex", pattern=eq, dimension=col)
|
||||||
|
elif op == '>=':
|
||||||
|
cond = Dimension(col) >= eq
|
||||||
|
elif op == '<=':
|
||||||
|
cond = Dimension(col) <= eq
|
||||||
|
elif op == '>':
|
||||||
|
cond = Dimension(col) > eq
|
||||||
|
elif op == '<':
|
||||||
|
cond = Dimension(col) < eq
|
||||||
if filters:
|
if filters:
|
||||||
filters = Filter(type="and", fields=[
|
filters = Filter(type="and", fields=[
|
||||||
cond,
|
cond,
|
||||||
|
|
|
@ -146,7 +146,8 @@ class DruidDatasourceModelView(SupersetModelView, DeleteMixin): # noqa
|
||||||
related_views = [DruidColumnInlineView, DruidMetricInlineView]
|
related_views = [DruidColumnInlineView, DruidMetricInlineView]
|
||||||
edit_columns = [
|
edit_columns = [
|
||||||
'datasource_name', 'cluster', 'description', 'owner',
|
'datasource_name', 'cluster', 'description', 'owner',
|
||||||
'is_featured', 'is_hidden', 'filter_select_enabled',
|
'is_featured', 'is_hidden',
|
||||||
|
'filter_select_enabled', 'fetch_values_from',
|
||||||
'default_endpoint', 'offset', 'cache_timeout']
|
'default_endpoint', 'offset', 'cache_timeout']
|
||||||
add_columns = edit_columns
|
add_columns = edit_columns
|
||||||
show_columns = add_columns + ['perm']
|
show_columns = add_columns + ['perm']
|
||||||
|
@ -157,6 +158,9 @@ class DruidDatasourceModelView(SupersetModelView, DeleteMixin): # noqa
|
||||||
'description': Markup(
|
'description': Markup(
|
||||||
"Supports <a href='"
|
"Supports <a href='"
|
||||||
"https://daringfireball.net/projects/markdown/'>markdown</a>"),
|
"https://daringfireball.net/projects/markdown/'>markdown</a>"),
|
||||||
|
'fetch_values_from': _(
|
||||||
|
"Time expression to use as a predicate when retrieving "
|
||||||
|
"distinct values to populate the filter component"),
|
||||||
}
|
}
|
||||||
base_filters = [['id', DatasourceFilter, lambda: []]]
|
base_filters = [['id', DatasourceFilter, lambda: []]]
|
||||||
label_columns = {
|
label_columns = {
|
||||||
|
|
|
@ -172,6 +172,7 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
|
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
|
||||||
is_featured = Column(Boolean, default=False)
|
is_featured = Column(Boolean, default=False)
|
||||||
filter_select_enabled = Column(Boolean, default=False)
|
filter_select_enabled = Column(Boolean, default=False)
|
||||||
|
fetch_values_predicate = Column(String(1000))
|
||||||
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
user_id = Column(Integer, ForeignKey('ab_user.id'))
|
||||||
owner = relationship('User', backref='tables', foreign_keys=[user_id])
|
owner = relationship('User', backref='tables', foreign_keys=[user_id])
|
||||||
database = relationship(
|
database = relationship(
|
||||||
|
@ -285,33 +286,25 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
if col_name == col.column_name:
|
if col_name == col.column_name:
|
||||||
return col
|
return col
|
||||||
|
|
||||||
def values_for_column(self,
|
def values_for_column(self, column_name, limit=10000):
|
||||||
column_name,
|
|
||||||
from_dttm,
|
|
||||||
to_dttm,
|
|
||||||
limit=500):
|
|
||||||
"""Runs query against sqla to retrieve some
|
"""Runs query against sqla to retrieve some
|
||||||
sample values for the given column.
|
sample values for the given column.
|
||||||
"""
|
"""
|
||||||
granularity = self.main_dttm_col
|
|
||||||
|
|
||||||
cols = {col.column_name: col for col in self.columns}
|
cols = {col.column_name: col for col in self.columns}
|
||||||
target_col = cols[column_name]
|
target_col = cols[column_name]
|
||||||
|
|
||||||
tbl = table(self.table_name)
|
tbl = table(self.table_name)
|
||||||
qry = sa.select([target_col.sqla_col])
|
qry = (
|
||||||
qry = qry.select_from(tbl)
|
select([target_col.sqla_col])
|
||||||
qry = qry.distinct(column_name)
|
.select_from(tbl)
|
||||||
|
.distinct(column_name)
|
||||||
|
)
|
||||||
|
if limit:
|
||||||
qry = qry.limit(limit)
|
qry = qry.limit(limit)
|
||||||
|
|
||||||
if granularity:
|
if self.fetch_values_predicate:
|
||||||
dttm_col = cols[granularity]
|
tp = self.get_template_processor()
|
||||||
timestamp = dttm_col.sqla_col.label('timestamp')
|
qry = qry.where(tp.process_template(self.fetch_values_predicate))
|
||||||
time_filter = [
|
|
||||||
timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)),
|
|
||||||
timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)),
|
|
||||||
]
|
|
||||||
qry = qry.where(and_(*time_filter))
|
|
||||||
|
|
||||||
engine = self.database.get_sqla_engine()
|
engine = self.database.get_sqla_engine()
|
||||||
sql = "{}".format(
|
sql = "{}".format(
|
||||||
|
@ -319,10 +312,15 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
engine, compile_kwargs={"literal_binds": True}, ),
|
engine, compile_kwargs={"literal_binds": True}, ),
|
||||||
)
|
)
|
||||||
|
|
||||||
return pd.read_sql_query(
|
df = pd.read_sql_query(
|
||||||
sql=sql,
|
sql=sql,
|
||||||
con=engine
|
con=engine
|
||||||
)
|
)
|
||||||
|
return [row[0] for row in df.to_records(index=False)]
|
||||||
|
|
||||||
|
def get_template_processor(self, **kwargs):
|
||||||
|
return get_template_processor(
|
||||||
|
table=self, database=self.database, **kwargs)
|
||||||
|
|
||||||
def get_query_str( # sqla
|
def get_query_str( # sqla
|
||||||
self, engine, qry_start_dttm,
|
self, engine, qry_start_dttm,
|
||||||
|
@ -340,6 +338,7 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
extras=None,
|
extras=None,
|
||||||
columns=None):
|
columns=None):
|
||||||
"""Querying any sqla table from this common interface"""
|
"""Querying any sqla table from this common interface"""
|
||||||
|
|
||||||
template_kwargs = {
|
template_kwargs = {
|
||||||
'from_dttm': from_dttm,
|
'from_dttm': from_dttm,
|
||||||
'groupby': groupby,
|
'groupby': groupby,
|
||||||
|
@ -347,8 +346,7 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
'row_limit': row_limit,
|
'row_limit': row_limit,
|
||||||
'to_dttm': to_dttm,
|
'to_dttm': to_dttm,
|
||||||
}
|
}
|
||||||
template_processor = get_template_processor(
|
template_processor = self.get_template_processor(**template_kwargs)
|
||||||
table=self, database=self.database, **template_kwargs)
|
|
||||||
|
|
||||||
# For backward compatibility
|
# For backward compatibility
|
||||||
if granularity not in self.dttm_cols:
|
if granularity not in self.dttm_cols:
|
||||||
|
@ -452,7 +450,8 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
op = flt['op']
|
op = flt['op']
|
||||||
eq = flt['val']
|
eq = flt['val']
|
||||||
col_obj = cols.get(col)
|
col_obj = cols.get(col)
|
||||||
if col_obj and op in ('in', 'not in'):
|
if col_obj:
|
||||||
|
if op in ('in', 'not in'):
|
||||||
values = [types.strip("'").strip('"') for types in eq]
|
values = [types.strip("'").strip('"') for types in eq]
|
||||||
if col_obj.is_num:
|
if col_obj.is_num:
|
||||||
values = [utils.js_string_to_num(s) for s in values]
|
values = [utils.js_string_to_num(s) for s in values]
|
||||||
|
@ -460,6 +459,21 @@ class SqlaTable(Model, BaseDatasource):
|
||||||
if op == 'not in':
|
if op == 'not in':
|
||||||
cond = ~cond
|
cond = ~cond
|
||||||
where_clause_and.append(cond)
|
where_clause_and.append(cond)
|
||||||
|
elif op == '==':
|
||||||
|
where_clause_and.append(col_obj.sqla_col == eq)
|
||||||
|
elif op == '!=':
|
||||||
|
where_clause_and.append(col_obj.sqla_col != eq)
|
||||||
|
elif op == '>':
|
||||||
|
where_clause_and.append(col_obj.sqla_col > eq)
|
||||||
|
elif op == '<':
|
||||||
|
where_clause_and.append(col_obj.sqla_col < eq)
|
||||||
|
elif op == '>=':
|
||||||
|
where_clause_and.append(col_obj.sqla_col >= eq)
|
||||||
|
elif op == '<=':
|
||||||
|
where_clause_and.append(col_obj.sqla_col <= eq)
|
||||||
|
elif op == 'LIKE':
|
||||||
|
where_clause_and.append(
|
||||||
|
col_obj.sqla_col.like(eq.replace('%', '%%')))
|
||||||
if extras:
|
if extras:
|
||||||
where = extras.get('where')
|
where = extras.get('where')
|
||||||
if where:
|
if where:
|
||||||
|
|
|
@ -136,7 +136,7 @@ class TableModelView(SupersetModelView, DeleteMixin): # noqa
|
||||||
add_columns = ['database', 'schema', 'table_name']
|
add_columns = ['database', 'schema', 'table_name']
|
||||||
edit_columns = [
|
edit_columns = [
|
||||||
'table_name', 'sql', 'is_featured', 'filter_select_enabled',
|
'table_name', 'sql', 'is_featured', 'filter_select_enabled',
|
||||||
'database', 'schema',
|
'fetch_values_predicate', 'database', 'schema',
|
||||||
'description', 'owner',
|
'description', 'owner',
|
||||||
'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
|
'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
|
||||||
show_columns = edit_columns + ['perm']
|
show_columns = edit_columns + ['perm']
|
||||||
|
@ -156,6 +156,11 @@ class TableModelView(SupersetModelView, DeleteMixin): # noqa
|
||||||
"This fields acts a Superset view, meaning that Superset will "
|
"This fields acts a Superset view, meaning that Superset will "
|
||||||
"run a query against this string as a subquery."
|
"run a query against this string as a subquery."
|
||||||
),
|
),
|
||||||
|
'fetch_values_predicate': _(
|
||||||
|
"Predicate applied when fetching distinct value to "
|
||||||
|
"populate the filter control component. Supports "
|
||||||
|
"jinja template syntax."
|
||||||
|
),
|
||||||
}
|
}
|
||||||
base_filters = [['id', DatasourceFilter, lambda: []]]
|
base_filters = [['id', DatasourceFilter, lambda: []]]
|
||||||
label_columns = {
|
label_columns = {
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
"""add fetch values predicate
|
||||||
|
|
||||||
|
Revision ID: 732f1c06bcbf
|
||||||
|
Revises: d6db5a5cdb5d
|
||||||
|
Create Date: 2017-03-03 09:15:56.800930
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '732f1c06bcbf'
|
||||||
|
down_revision = 'd6db5a5cdb5d'
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.add_column('datasources', sa.Column('fetch_values_from', sa.String(length=100), nullable=True))
|
||||||
|
op.add_column('tables', sa.Column('fetch_values_predicate', sa.String(length=1000), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_column('tables', 'fetch_values_predicate')
|
||||||
|
op.drop_column('datasources', 'fetch_values_from')
|
|
@ -0,0 +1,22 @@
|
||||||
|
"""empty message
|
||||||
|
|
||||||
|
Revision ID: ea033256294a
|
||||||
|
Revises: ('732f1c06bcbf', 'b318dfe5fb6c')
|
||||||
|
Create Date: 2017-03-16 14:55:59.431283
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = 'ea033256294a'
|
||||||
|
down_revision = ('732f1c06bcbf', 'b318dfe5fb6c')
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
pass
|
|
@ -1128,33 +1128,17 @@ class Superset(BaseSupersetView):
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
# TODO: Cache endpoint by user, datasource and column
|
# TODO: Cache endpoint by user, datasource and column
|
||||||
error_redirect = '/slicemodelview/list/'
|
|
||||||
datasource_class = ConnectorRegistry.sources[datasource_type]
|
datasource_class = ConnectorRegistry.sources[datasource_type]
|
||||||
|
|
||||||
datasource = db.session.query(
|
datasource = db.session.query(
|
||||||
datasource_class).filter_by(id=datasource_id).first()
|
datasource_class).filter_by(id=datasource_id).first()
|
||||||
|
|
||||||
if not datasource:
|
if not datasource:
|
||||||
flash(DATASOURCE_MISSING_ERR, "alert")
|
|
||||||
return json_error_response(DATASOURCE_MISSING_ERR)
|
return json_error_response(DATASOURCE_MISSING_ERR)
|
||||||
if not self.datasource_access(datasource):
|
if not self.datasource_access(datasource):
|
||||||
flash(get_datasource_access_error_msg(datasource.name), "danger")
|
|
||||||
return json_error_response(DATASOURCE_ACCESS_ERR)
|
return json_error_response(DATASOURCE_ACCESS_ERR)
|
||||||
|
|
||||||
viz_type = request.args.get("viz_type")
|
payload = json.dumps(datasource.values_for_column(column))
|
||||||
if not viz_type and datasource.default_endpoint:
|
return json_success(payload)
|
||||||
return redirect(datasource.default_endpoint)
|
|
||||||
if not viz_type:
|
|
||||||
viz_type = "table"
|
|
||||||
try:
|
|
||||||
obj = viz.viz_types[viz_type](
|
|
||||||
datasource,
|
|
||||||
form_data=request.args,
|
|
||||||
slice_=None)
|
|
||||||
except Exception as e:
|
|
||||||
flash(str(e), "danger")
|
|
||||||
return redirect(error_redirect)
|
|
||||||
return json_success(obj.get_values_for_column(column))
|
|
||||||
|
|
||||||
def save_or_overwrite_slice(
|
def save_or_overwrite_slice(
|
||||||
self, args, slc, slice_add_perm, slice_overwrite_perm,
|
self, args, slc, slice_add_perm, slice_overwrite_perm,
|
||||||
|
|
|
@ -45,7 +45,6 @@ class BaseViz(object):
|
||||||
is_timeseries = False
|
is_timeseries = False
|
||||||
|
|
||||||
def __init__(self, datasource, form_data, slice_=None):
|
def __init__(self, datasource, form_data, slice_=None):
|
||||||
self.orig_form_data = form_data
|
|
||||||
if not datasource:
|
if not datasource:
|
||||||
raise Exception("Viz is missing a datasource")
|
raise Exception("Viz is missing a datasource")
|
||||||
self.datasource = datasource
|
self.datasource = datasource
|
||||||
|
@ -63,34 +62,6 @@ class BaseViz(object):
|
||||||
self.status = None
|
self.status = None
|
||||||
self.error_message = None
|
self.error_message = None
|
||||||
|
|
||||||
def get_filter_url(self):
|
|
||||||
"""Returns the URL to retrieve column values used in the filter"""
|
|
||||||
data = self.orig_form_data.copy()
|
|
||||||
# Remove unchecked checkboxes because HTML is weird like that
|
|
||||||
ordered_data = MultiDict()
|
|
||||||
for key in sorted(data.keys()):
|
|
||||||
# if MultiDict is initialized with MD({key:[emptyarray]}),
|
|
||||||
# key is included in d.keys() but accessing it throws
|
|
||||||
try:
|
|
||||||
if data[key] is False:
|
|
||||||
del data[key]
|
|
||||||
continue
|
|
||||||
except IndexError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if isinstance(data, (MultiDict, ImmutableMultiDict)):
|
|
||||||
v = data.getlist(key)
|
|
||||||
else:
|
|
||||||
v = data.get(key)
|
|
||||||
if not isinstance(v, list):
|
|
||||||
v = [v]
|
|
||||||
for item in v:
|
|
||||||
ordered_data.add(key, item)
|
|
||||||
href = Href(
|
|
||||||
'/superset/filter/{self.datasource.type}/'
|
|
||||||
'{self.datasource.id}/'.format(**locals()))
|
|
||||||
return href(ordered_data)
|
|
||||||
|
|
||||||
def get_df(self, query_obj=None):
|
def get_df(self, query_obj=None):
|
||||||
"""Returns a pandas dataframe based on the query object"""
|
"""Returns a pandas dataframe based on the query object"""
|
||||||
if not query_obj:
|
if not query_obj:
|
||||||
|
@ -277,7 +248,6 @@ class BaseViz(object):
|
||||||
'cache_timeout': cache_timeout,
|
'cache_timeout': cache_timeout,
|
||||||
'data': data,
|
'data': data,
|
||||||
'error': self.error_message,
|
'error': self.error_message,
|
||||||
'filter_endpoint': self.filter_endpoint,
|
|
||||||
'form_data': self.form_data,
|
'form_data': self.form_data,
|
||||||
'query': self.query,
|
'query': self.query,
|
||||||
'status': self.status,
|
'status': self.status,
|
||||||
|
@ -312,7 +282,6 @@ class BaseViz(object):
|
||||||
"""This is the data object serialized to the js layer"""
|
"""This is the data object serialized to the js layer"""
|
||||||
content = {
|
content = {
|
||||||
'form_data': self.form_data,
|
'form_data': self.form_data,
|
||||||
'filter_endpoint': self.filter_endpoint,
|
|
||||||
'token': self.token,
|
'token': self.token,
|
||||||
'viz_name': self.viz_type,
|
'viz_name': self.viz_type,
|
||||||
'filter_select_enabled': self.datasource.filter_select_enabled,
|
'filter_select_enabled': self.datasource.filter_select_enabled,
|
||||||
|
@ -324,40 +293,9 @@ class BaseViz(object):
|
||||||
include_index = not isinstance(df.index, pd.RangeIndex)
|
include_index = not isinstance(df.index, pd.RangeIndex)
|
||||||
return df.to_csv(index=include_index, encoding="utf-8")
|
return df.to_csv(index=include_index, encoding="utf-8")
|
||||||
|
|
||||||
def get_values_for_column(self, column):
|
|
||||||
"""
|
|
||||||
Retrieves values for a column to be used by the filter dropdown.
|
|
||||||
|
|
||||||
:param column: column name
|
|
||||||
:return: JSON containing the some values for a column
|
|
||||||
"""
|
|
||||||
form_data = self.form_data
|
|
||||||
|
|
||||||
since = form_data.get("since", "1 year ago")
|
|
||||||
from_dttm = utils.parse_human_datetime(since)
|
|
||||||
now = datetime.now()
|
|
||||||
if from_dttm > now:
|
|
||||||
from_dttm = now - (from_dttm - now)
|
|
||||||
until = form_data.get("until", "now")
|
|
||||||
to_dttm = utils.parse_human_datetime(until)
|
|
||||||
if from_dttm > to_dttm:
|
|
||||||
raise Exception("From date cannot be larger than to date")
|
|
||||||
|
|
||||||
kwargs = dict(
|
|
||||||
column_name=column,
|
|
||||||
from_dttm=from_dttm,
|
|
||||||
to_dttm=to_dttm,
|
|
||||||
)
|
|
||||||
df = self.datasource.values_for_column(**kwargs)
|
|
||||||
return df[column].to_json()
|
|
||||||
|
|
||||||
def get_data(self, df):
|
def get_data(self, df):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@property
|
|
||||||
def filter_endpoint(self):
|
|
||||||
return self.get_filter_url()
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def json_data(self):
|
def json_data(self):
|
||||||
return json.dumps(self.data)
|
return json.dumps(self.data)
|
||||||
|
|
Loading…
Reference in New Issue