[explore] improved filters (#2330)

* Support more filter operators

* more filter operators [>, <, >=, <=, ==, !=, LIKE]
* Fix need to escape/double `%` in LIKE clauses
* spinner while loading values when changing column
* datasource config elements to allow to applying predicates when
  fetching filter values
* refactor

* Removing doubling parens

* rebasing

* Merging migrations
This commit is contained in:
Maxime Beauchemin 2017-03-20 21:10:59 -07:00 committed by GitHub
parent 82bc907088
commit 8042ac876e
15 changed files with 245 additions and 243 deletions

View File

@ -67,7 +67,6 @@ setup(
'sqlparse==0.1.19', 'sqlparse==0.1.19',
'thrift>=0.9.3', 'thrift>=0.9.3',
'thrift-sasl>=0.2.1', 'thrift-sasl>=0.2.1',
'werkzeug==0.11.15',
], ],
extras_require={ extras_require={
'cors': ['Flask-Cors>=2.0.0'], 'cors': ['Flask-Cors>=2.0.0'],

View File

@ -4,8 +4,22 @@ import Select from 'react-select';
import { Button, Row, Col } from 'react-bootstrap'; import { Button, Row, Col } from 'react-bootstrap';
import SelectControl from './SelectControl'; import SelectControl from './SelectControl';
const arrayFilterOps = ['in', 'not in']; const operatorsArr = [
const strFilterOps = ['==', '!=', '>', '<', '>=', '<=', 'regex']; { val: 'in', type: 'array', useSelect: true, multi: true },
{ val: 'not in', type: 'array', useSelect: true, multi: true },
{ val: '==', type: 'string', useSelect: true, multi: false },
{ val: '!=', type: 'string', useSelect: true, multi: false },
{ val: '>=', type: 'string' },
{ val: '<=', type: 'string' },
{ val: '>', type: 'string' },
{ val: '<', type: 'string' },
{ val: 'regex', type: 'string', datasourceTypes: ['druid'] },
{ val: 'LIKE', type: 'string', datasourceTypes: ['table'] },
];
const operators = {};
operatorsArr.forEach(op => {
operators[op.val] = op;
});
const propTypes = { const propTypes = {
choices: PropTypes.array, choices: PropTypes.array,
@ -13,11 +27,9 @@ const propTypes = {
removeFilter: PropTypes.func, removeFilter: PropTypes.func,
filter: PropTypes.object.isRequired, filter: PropTypes.object.isRequired,
datasource: PropTypes.object, datasource: PropTypes.object,
having: PropTypes.bool,
}; };
const defaultProps = { const defaultProps = {
having: false,
changeFilter: () => {}, changeFilter: () => {},
removeFilter: () => {}, removeFilter: () => {},
choices: [], choices: [],
@ -27,102 +39,91 @@ const defaultProps = {
export default class Filter extends React.Component { export default class Filter extends React.Component {
constructor(props) { constructor(props) {
super(props); super(props);
const filterOps = props.datasource.type === 'table' ? this.state = {
['in', 'not in'] : ['==', '!=', 'in', 'not in', 'regex']; valuesLoading: false,
this.opChoices = this.props.having ? ['==', '!=', '>', '<', '>=', '<='] };
: filterOps; }
componentDidMount() {
this.fetchFilterValues(this.props.filter.col);
} }
fetchFilterValues(col) { fetchFilterValues(col) {
if (!this.props.datasource) {
return;
}
const datasource = this.props.datasource; const datasource = this.props.datasource;
let choices = []; if (col && this.props.datasource && this.props.datasource.filter_select) {
if (col) { this.setState({ valuesLoading: true });
$.ajax({ $.ajax({
type: 'GET', type: 'GET',
url: `/superset/filter/${datasource.type}/${datasource.id}/${col}/`, url: `/superset/filter/${datasource.type}/${datasource.id}/${col}/`,
success: (data) => { success: (data) => {
choices = Object.keys(data).map((k) => this.props.changeFilter('choices', data);
([`'${data[k]}'`, `'${data[k]}'`])); this.setState({ valuesLoading: false });
this.props.changeFilter('choices', choices);
}, },
}); });
} }
} }
switchFilterValue(prevFilter, nextOp) { switchFilterValue(prevOp, nextOp) {
const prevOp = prevFilter.op; if (operators[prevOp].type !== operators[nextOp].type) {
let newVal = null; const val = this.props.filter.value;
if (arrayFilterOps.indexOf(prevOp) !== -1 let newVal;
&& strFilterOps.indexOf(nextOp) !== -1) {
// switch from array to string // switch from array to string
newVal = this.props.filter.val.length > 0 ? this.props.filter.val[0] : ''; if (operators[nextOp].type === 'string' && val && val.length > 0) {
} newVal = val[0];
if (strFilterOps.indexOf(prevOp) !== -1 } else if (operators[nextOp].type === 'string' && val) {
&& arrayFilterOps.indexOf(nextOp) !== -1) { newVal = [val];
// switch from string to array
newVal = this.props.filter.val === '' ? [] : [this.props.filter.val];
}
return newVal;
}
changeFilter(control, event) {
let value = event;
if (event && event.target) {
value = event.target.value;
}
if (event && event.value) {
value = event.value;
}
if (control === 'op') {
const newVal = this.switchFilterValue(this.props.filter, value);
if (newVal) {
this.props.changeFilter(['op', 'val'], [value, newVal]);
} else {
this.props.changeFilter(control, value);
} }
} else { this.props.changeFilter('val', newVal);
this.props.changeFilter(control, value);
}
if (control === 'col' && value !== null && this.props.datasource.filter_select) {
this.fetchFilterValues(value);
} }
} }
changeText(event) {
this.props.changeFilter('val', event.target.value);
}
changeSelect(value) {
this.props.changeFilter('val', value);
}
changeColumn(event) {
this.props.changeFilter('col', event.value);
this.fetchFilterValues(event.value);
}
changeOp(event) {
this.switchFilterValue(this.props.filter.op, event.value);
this.props.changeFilter('op', event.value);
}
removeFilter(filter) { removeFilter(filter) {
this.props.removeFilter(filter); this.props.removeFilter(filter);
} }
renderFilterFormControl(filter) { renderFilterFormControl(filter) {
const datasource = this.props.datasource; const operator = operators[filter.op];
if (datasource && datasource.filter_select) { if (operator.useSelect) {
if (!filter.choices) {
this.fetchFilterValues(filter.col);
}
}
// switching filter value between array/string when needed
if (strFilterOps.indexOf(filter.op) !== -1) {
// druid having filter or regex/==/!= filters
return ( return (
<input <SelectControl
type="text" multi={operator.multi}
onChange={this.changeFilter.bind(this, 'val')} freeForm
name="filter-value"
value={filter.val} value={filter.val}
className="form-control input-sm" isLoading={this.state.valuesLoading}
placeholder="Filter value" choices={filter.choices}
onChange={this.changeSelect.bind(this)}
/> />
); );
} }
return ( return (
<SelectControl <input
multi type="text"
freeForm onChange={this.changeText.bind(this)}
name="filter-value"
value={filter.val} value={filter.val}
choices={filter.choices || []} className="form-control input-sm"
onChange={this.changeFilter.bind(this, 'val')} placeholder="Filter value"
/> />
); );
} }
render() { render() {
const datasource = this.props.datasource;
const filter = this.props.filter; const filter = this.props.filter;
const opsChoices = operatorsArr
.filter(o => !o.datasourceTypes || o.datasourceTypes.indexOf(datasource.type) >= 0)
.map(o => ({ value: o.val, label: o.val }));
const colChoices = datasource ?
datasource.filterable_cols.map(c => ({ value: c[0], label: c[1] })) :
null;
return ( return (
<div> <div>
<Row className="space-1"> <Row className="space-1">
@ -130,9 +131,10 @@ export default class Filter extends React.Component {
<Select <Select
id="select-col" id="select-col"
placeholder="Select column" placeholder="Select column"
options={this.props.choices.map((c) => ({ value: c[0], label: c[1] }))} clearable={false}
options={colChoices}
value={filter.col} value={filter.col}
onChange={this.changeFilter.bind(this, 'col')} onChange={this.changeColumn.bind(this)}
/> />
</Col> </Col>
</Row> </Row>
@ -141,9 +143,10 @@ export default class Filter extends React.Component {
<Select <Select
id="select-op" id="select-op"
placeholder="Select operator" placeholder="Select operator"
options={this.opChoices.map((o) => ({ value: o, label: o }))} options={opsChoices}
clearable={false}
value={filter.op} value={filter.op}
onChange={this.changeFilter.bind(this, 'op')} onChange={this.changeOp.bind(this)}
/> />
</Col> </Col>
<Col md={7}> <Col md={7}>

View File

@ -4,14 +4,12 @@ import Filter from './Filter';
const propTypes = { const propTypes = {
name: PropTypes.string, name: PropTypes.string,
choices: PropTypes.array,
onChange: PropTypes.func, onChange: PropTypes.func,
value: PropTypes.array, value: PropTypes.array,
datasource: PropTypes.object, datasource: PropTypes.object,
}; };
const defaultProps = { const defaultProps = {
choices: [],
onChange: () => {}, onChange: () => {},
value: [], value: [],
}; };
@ -19,8 +17,11 @@ const defaultProps = {
export default class FilterControl extends React.Component { export default class FilterControl extends React.Component {
addFilter() { addFilter() {
const newFilters = Object.assign([], this.props.value); const newFilters = Object.assign([], this.props.value);
const col = this.props.datasource && this.props.datasource.filterable_cols.length > 0 ?
this.props.datasource.filterable_cols[0][0] :
null;
newFilters.push({ newFilters.push({
col: null, col,
op: 'in', op: 'in',
val: this.props.datasource.filter_select ? [] : '', val: this.props.datasource.filter_select ? [] : '',
}); });
@ -43,22 +44,17 @@ export default class FilterControl extends React.Component {
this.props.onChange(this.props.value.filter((f, i) => i !== index)); this.props.onChange(this.props.value.filter((f, i) => i !== index));
} }
render() { render() {
const filters = []; const filters = this.props.value.map((filter, i) => (
this.props.value.forEach((filter, i) => { <div key={i}>
const filterBox = ( <Filter
<div key={i}> having={this.props.name === 'having_filters'}
<Filter filter={filter}
having={this.props.name === 'having_filters'} datasource={this.props.datasource}
filter={filter} removeFilter={this.removeFilter.bind(this, i)}
choices={this.props.choices} changeFilter={this.changeFilter.bind(this, i)}
datasource={this.props.datasource} />
removeFilter={this.removeFilter.bind(this, i)} </div>
changeFilter={this.changeFilter.bind(this, i)} ));
/>
</div>
);
filters.push(filterBox);
});
return ( return (
<div> <div>
{filters} {filters}

View File

@ -47,30 +47,39 @@ export default class SelectControl extends React.PureComponent {
this.props.onChange(optionValue); this.props.onChange(optionValue);
} }
getOptions(props) { getOptions(props) {
const options = props.choices.map((c) => { // Accepts different formats of input
const label = c.length > 1 ? c[1] : c[0]; const options = props.choices.map(c => {
const newOptions = { let option;
value: c[0], if (Array.isArray(c)) {
label, const label = c.length > 1 ? c[1] : c[0];
}; option = {
if (c[2]) newOptions.imgSrc = c[2]; value: c[0],
return newOptions; label,
};
if (c[2]) option.imgSrc = c[2];
} else if (Object.is(c)) {
option = c;
} else {
option = {
value: c,
label: c,
};
}
return option;
}); });
if (props.freeForm) { if (props.freeForm) {
// For FreeFormSelect, insert value into options if not exist // For FreeFormSelect, insert value into options if not exist
const values = props.choices.map((c) => c[0]); const values = options.map(c => c.value);
if (props.value) { if (props.value) {
if (typeof props.value === 'object') { let valuesToAdd = props.value;
props.value.forEach((v) => { if (!Array.isArray(valuesToAdd)) {
if (values.indexOf(v) === -1) { valuesToAdd = [valuesToAdd];
options.push({ value: v, label: v });
}
});
} else {
if (values.indexOf(props.value) === -1) {
options.push({ value: props.value, label: props.value });
}
} }
valuesToAdd.forEach(v => {
if (values.indexOf(v) < 0) {
options.push({ value: v, label: v });
}
});
} }
} }
return options; return options;

View File

@ -1177,7 +1177,6 @@ export const controls = {
default: [], default: [],
description: '', description: '',
mapStateToProps: (state) => ({ mapStateToProps: (state) => ({
choices: (state.datasource) ? state.datasource.filterable_cols : [],
datasource: state.datasource, datasource: state.datasource,
}), }),
}, },

View File

@ -17,6 +17,7 @@ const defaultProps = {
id: 1, id: 1,
type: 'table', type: 'table',
filter_select: false, filter_select: false,
filterable_cols: ['country_name'],
}, },
}; };

View File

@ -10,11 +10,8 @@ import Filter from '../../../../javascripts/explorev2/components/controls/Filter
import SelectControl from '../../../../javascripts/explorev2/components/controls/SelectControl'; import SelectControl from '../../../../javascripts/explorev2/components/controls/SelectControl';
const defaultProps = { const defaultProps = {
choices: ['country_name'],
changeFilter: sinon.spy(), changeFilter: sinon.spy(),
removeFilter: () => { removeFilter: () => {},
// noop
},
filter: { filter: {
col: null, col: null,
op: 'in', op: 'in',
@ -22,8 +19,9 @@ const defaultProps = {
}, },
datasource: { datasource: {
id: 1, id: 1,
type: 'table', type: 'qtable',
filter_select: false, filter_select: false,
filterable_cols: ['col1', 'col2'],
}, },
}; };
@ -44,7 +42,7 @@ describe('Filter', () => {
expect(wrapper.find(Select)).to.have.lengthOf(2); expect(wrapper.find(Select)).to.have.lengthOf(2);
expect(wrapper.find(Button)).to.have.lengthOf(1); expect(wrapper.find(Button)).to.have.lengthOf(1);
expect(wrapper.find(SelectControl)).to.have.lengthOf(1); expect(wrapper.find(SelectControl)).to.have.lengthOf(1);
expect(wrapper.find('#select-op').prop('options')).to.have.lengthOf(2); expect(wrapper.find('#select-op').prop('options')).to.have.lengthOf(8);
}); });
it('renders five op choices for table datasource', () => { it('renders five op choices for table datasource', () => {
@ -53,16 +51,17 @@ describe('Filter', () => {
id: 1, id: 1,
type: 'druid', type: 'druid',
filter_select: false, filter_select: false,
filterable_cols: ['country_name'],
}; };
const druidWrapper = shallow(<Filter {...props} />); const druidWrapper = shallow(<Filter {...props} />);
expect(druidWrapper.find('#select-op').prop('options')).to.have.lengthOf(5); expect(druidWrapper.find('#select-op').prop('options')).to.have.lengthOf(9);
}); });
it('renders six op choices for having filter', () => { it('renders six op choices for having filter', () => {
const props = defaultProps; const props = defaultProps;
props.having = true; props.having = true;
const havingWrapper = shallow(<Filter {...props} />); const havingWrapper = shallow(<Filter {...props} />);
expect(havingWrapper.find('#select-op').prop('options')).to.have.lengthOf(6); expect(havingWrapper.find('#select-op').prop('options')).to.have.lengthOf(9);
}); });
it('calls changeFilter when select is changed', () => { it('calls changeFilter when select is changed', () => {

View File

@ -326,6 +326,7 @@ class DruidDatasource(Model, BaseDatasource):
is_hidden = Column(Boolean, default=False) is_hidden = Column(Boolean, default=False)
filter_select_enabled = Column(Boolean, default=False) filter_select_enabled = Column(Boolean, default=False)
description = Column(Text) description = Column(Text)
fetch_values_from = Column(String(100))
default_endpoint = Column(Text) default_endpoint = Column(Text)
user_id = Column(Integer, ForeignKey('ab_user.id')) user_id = Column(Integer, ForeignKey('ab_user.id'))
owner = relationship( owner = relationship(
@ -696,18 +697,18 @@ class DruidDatasource(Model, BaseDatasource):
def values_for_column(self, def values_for_column(self,
column_name, column_name,
from_dttm, limit=10000):
to_dttm,
limit=500):
"""Retrieve some values for the given column""" """Retrieve some values for the given column"""
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid # TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
from_dttm = from_dttm.replace(tzinfo=DRUID_TZ) if self.fetch_values_from:
to_dttm = to_dttm.replace(tzinfo=DRUID_TZ) from_dttm = utils.parse_human_datetime(self.fetch_values_from)
else:
from_dttm = datetime(1970, 1, 1)
qry = dict( qry = dict(
datasource=self.datasource_name, datasource=self.datasource_name,
granularity="all", granularity="all",
intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(), intervals=from_dttm.isoformat() + '/' + datetime.now().isoformat(),
aggregations=dict(count=count("count")), aggregations=dict(count=count("count")),
dimension=column_name, dimension=column_name,
metric="count", metric="count",
@ -718,10 +719,7 @@ class DruidDatasource(Model, BaseDatasource):
client.topn(**qry) client.topn(**qry)
df = client.export_pandas() df = client.export_pandas()
if df is None or df.size == 0: return [row[0] for row in df.to_records(index=False)]
raise Exception(_("No data was returned."))
return df
def get_query_str( # noqa / druid def get_query_str( # noqa / druid
self, client, qry_start_dttm, self, client, qry_start_dttm,
@ -1015,6 +1013,14 @@ class DruidDatasource(Model, BaseDatasource):
cond = ~cond cond = ~cond
elif op == 'regex': elif op == 'regex':
cond = Filter(type="regex", pattern=eq, dimension=col) cond = Filter(type="regex", pattern=eq, dimension=col)
elif op == '>=':
cond = Dimension(col) >= eq
elif op == '<=':
cond = Dimension(col) <= eq
elif op == '>':
cond = Dimension(col) > eq
elif op == '<':
cond = Dimension(col) < eq
if filters: if filters:
filters = Filter(type="and", fields=[ filters = Filter(type="and", fields=[
cond, cond,

View File

@ -146,7 +146,8 @@ class DruidDatasourceModelView(SupersetModelView, DeleteMixin): # noqa
related_views = [DruidColumnInlineView, DruidMetricInlineView] related_views = [DruidColumnInlineView, DruidMetricInlineView]
edit_columns = [ edit_columns = [
'datasource_name', 'cluster', 'description', 'owner', 'datasource_name', 'cluster', 'description', 'owner',
'is_featured', 'is_hidden', 'filter_select_enabled', 'is_featured', 'is_hidden',
'filter_select_enabled', 'fetch_values_from',
'default_endpoint', 'offset', 'cache_timeout'] 'default_endpoint', 'offset', 'cache_timeout']
add_columns = edit_columns add_columns = edit_columns
show_columns = add_columns + ['perm'] show_columns = add_columns + ['perm']
@ -157,6 +158,9 @@ class DruidDatasourceModelView(SupersetModelView, DeleteMixin): # noqa
'description': Markup( 'description': Markup(
"Supports <a href='" "Supports <a href='"
"https://daringfireball.net/projects/markdown/'>markdown</a>"), "https://daringfireball.net/projects/markdown/'>markdown</a>"),
'fetch_values_from': _(
"Time expression to use as a predicate when retrieving "
"distinct values to populate the filter component"),
} }
base_filters = [['id', DatasourceFilter, lambda: []]] base_filters = [['id', DatasourceFilter, lambda: []]]
label_columns = { label_columns = {

View File

@ -172,6 +172,7 @@ class SqlaTable(Model, BaseDatasource):
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
is_featured = Column(Boolean, default=False) is_featured = Column(Boolean, default=False)
filter_select_enabled = Column(Boolean, default=False) filter_select_enabled = Column(Boolean, default=False)
fetch_values_predicate = Column(String(1000))
user_id = Column(Integer, ForeignKey('ab_user.id')) user_id = Column(Integer, ForeignKey('ab_user.id'))
owner = relationship('User', backref='tables', foreign_keys=[user_id]) owner = relationship('User', backref='tables', foreign_keys=[user_id])
database = relationship( database = relationship(
@ -285,33 +286,25 @@ class SqlaTable(Model, BaseDatasource):
if col_name == col.column_name: if col_name == col.column_name:
return col return col
def values_for_column(self, def values_for_column(self, column_name, limit=10000):
column_name,
from_dttm,
to_dttm,
limit=500):
"""Runs query against sqla to retrieve some """Runs query against sqla to retrieve some
sample values for the given column. sample values for the given column.
""" """
granularity = self.main_dttm_col
cols = {col.column_name: col for col in self.columns} cols = {col.column_name: col for col in self.columns}
target_col = cols[column_name] target_col = cols[column_name]
tbl = table(self.table_name) tbl = table(self.table_name)
qry = sa.select([target_col.sqla_col]) qry = (
qry = qry.select_from(tbl) select([target_col.sqla_col])
qry = qry.distinct(column_name) .select_from(tbl)
qry = qry.limit(limit) .distinct(column_name)
)
if limit:
qry = qry.limit(limit)
if granularity: if self.fetch_values_predicate:
dttm_col = cols[granularity] tp = self.get_template_processor()
timestamp = dttm_col.sqla_col.label('timestamp') qry = qry.where(tp.process_template(self.fetch_values_predicate))
time_filter = [
timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)),
timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)),
]
qry = qry.where(and_(*time_filter))
engine = self.database.get_sqla_engine() engine = self.database.get_sqla_engine()
sql = "{}".format( sql = "{}".format(
@ -319,10 +312,15 @@ class SqlaTable(Model, BaseDatasource):
engine, compile_kwargs={"literal_binds": True}, ), engine, compile_kwargs={"literal_binds": True}, ),
) )
return pd.read_sql_query( df = pd.read_sql_query(
sql=sql, sql=sql,
con=engine con=engine
) )
return [row[0] for row in df.to_records(index=False)]
def get_template_processor(self, **kwargs):
return get_template_processor(
table=self, database=self.database, **kwargs)
def get_query_str( # sqla def get_query_str( # sqla
self, engine, qry_start_dttm, self, engine, qry_start_dttm,
@ -340,6 +338,7 @@ class SqlaTable(Model, BaseDatasource):
extras=None, extras=None,
columns=None): columns=None):
"""Querying any sqla table from this common interface""" """Querying any sqla table from this common interface"""
template_kwargs = { template_kwargs = {
'from_dttm': from_dttm, 'from_dttm': from_dttm,
'groupby': groupby, 'groupby': groupby,
@ -347,8 +346,7 @@ class SqlaTable(Model, BaseDatasource):
'row_limit': row_limit, 'row_limit': row_limit,
'to_dttm': to_dttm, 'to_dttm': to_dttm,
} }
template_processor = get_template_processor( template_processor = self.get_template_processor(**template_kwargs)
table=self, database=self.database, **template_kwargs)
# For backward compatibility # For backward compatibility
if granularity not in self.dttm_cols: if granularity not in self.dttm_cols:
@ -452,14 +450,30 @@ class SqlaTable(Model, BaseDatasource):
op = flt['op'] op = flt['op']
eq = flt['val'] eq = flt['val']
col_obj = cols.get(col) col_obj = cols.get(col)
if col_obj and op in ('in', 'not in'): if col_obj:
values = [types.strip("'").strip('"') for types in eq] if op in ('in', 'not in'):
if col_obj.is_num: values = [types.strip("'").strip('"') for types in eq]
values = [utils.js_string_to_num(s) for s in values] if col_obj.is_num:
cond = col_obj.sqla_col.in_(values) values = [utils.js_string_to_num(s) for s in values]
if op == 'not in': cond = col_obj.sqla_col.in_(values)
cond = ~cond if op == 'not in':
where_clause_and.append(cond) cond = ~cond
where_clause_and.append(cond)
elif op == '==':
where_clause_and.append(col_obj.sqla_col == eq)
elif op == '!=':
where_clause_and.append(col_obj.sqla_col != eq)
elif op == '>':
where_clause_and.append(col_obj.sqla_col > eq)
elif op == '<':
where_clause_and.append(col_obj.sqla_col < eq)
elif op == '>=':
where_clause_and.append(col_obj.sqla_col >= eq)
elif op == '<=':
where_clause_and.append(col_obj.sqla_col <= eq)
elif op == 'LIKE':
where_clause_and.append(
col_obj.sqla_col.like(eq.replace('%', '%%')))
if extras: if extras:
where = extras.get('where') where = extras.get('where')
if where: if where:

View File

@ -136,7 +136,7 @@ class TableModelView(SupersetModelView, DeleteMixin): # noqa
add_columns = ['database', 'schema', 'table_name'] add_columns = ['database', 'schema', 'table_name']
edit_columns = [ edit_columns = [
'table_name', 'sql', 'is_featured', 'filter_select_enabled', 'table_name', 'sql', 'is_featured', 'filter_select_enabled',
'database', 'schema', 'fetch_values_predicate', 'database', 'schema',
'description', 'owner', 'description', 'owner',
'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout'] 'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
show_columns = edit_columns + ['perm'] show_columns = edit_columns + ['perm']
@ -156,6 +156,11 @@ class TableModelView(SupersetModelView, DeleteMixin): # noqa
"This fields acts a Superset view, meaning that Superset will " "This fields acts a Superset view, meaning that Superset will "
"run a query against this string as a subquery." "run a query against this string as a subquery."
), ),
'fetch_values_predicate': _(
"Predicate applied when fetching distinct value to "
"populate the filter control component. Supports "
"jinja template syntax."
),
} }
base_filters = [['id', DatasourceFilter, lambda: []]] base_filters = [['id', DatasourceFilter, lambda: []]]
label_columns = { label_columns = {

View File

@ -0,0 +1,23 @@
"""add fetch values predicate
Revision ID: 732f1c06bcbf
Revises: d6db5a5cdb5d
Create Date: 2017-03-03 09:15:56.800930
"""
# revision identifiers, used by Alembic.
revision = '732f1c06bcbf'
down_revision = 'd6db5a5cdb5d'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('datasources', sa.Column('fetch_values_from', sa.String(length=100), nullable=True))
op.add_column('tables', sa.Column('fetch_values_predicate', sa.String(length=1000), nullable=True))
def downgrade():
op.drop_column('tables', 'fetch_values_predicate')
op.drop_column('datasources', 'fetch_values_from')

View File

@ -0,0 +1,22 @@
"""empty message
Revision ID: ea033256294a
Revises: ('732f1c06bcbf', 'b318dfe5fb6c')
Create Date: 2017-03-16 14:55:59.431283
"""
# revision identifiers, used by Alembic.
revision = 'ea033256294a'
down_revision = ('732f1c06bcbf', 'b318dfe5fb6c')
from alembic import op
import sqlalchemy as sa
def upgrade():
pass
def downgrade():
pass

View File

@ -1128,33 +1128,17 @@ class Superset(BaseSupersetView):
:return: :return:
""" """
# TODO: Cache endpoint by user, datasource and column # TODO: Cache endpoint by user, datasource and column
error_redirect = '/slicemodelview/list/'
datasource_class = ConnectorRegistry.sources[datasource_type] datasource_class = ConnectorRegistry.sources[datasource_type]
datasource = db.session.query( datasource = db.session.query(
datasource_class).filter_by(id=datasource_id).first() datasource_class).filter_by(id=datasource_id).first()
if not datasource: if not datasource:
flash(DATASOURCE_MISSING_ERR, "alert")
return json_error_response(DATASOURCE_MISSING_ERR) return json_error_response(DATASOURCE_MISSING_ERR)
if not self.datasource_access(datasource): if not self.datasource_access(datasource):
flash(get_datasource_access_error_msg(datasource.name), "danger")
return json_error_response(DATASOURCE_ACCESS_ERR) return json_error_response(DATASOURCE_ACCESS_ERR)
viz_type = request.args.get("viz_type") payload = json.dumps(datasource.values_for_column(column))
if not viz_type and datasource.default_endpoint: return json_success(payload)
return redirect(datasource.default_endpoint)
if not viz_type:
viz_type = "table"
try:
obj = viz.viz_types[viz_type](
datasource,
form_data=request.args,
slice_=None)
except Exception as e:
flash(str(e), "danger")
return redirect(error_redirect)
return json_success(obj.get_values_for_column(column))
def save_or_overwrite_slice( def save_or_overwrite_slice(
self, args, slc, slice_add_perm, slice_overwrite_perm, self, args, slc, slice_add_perm, slice_overwrite_perm,

View File

@ -45,7 +45,6 @@ class BaseViz(object):
is_timeseries = False is_timeseries = False
def __init__(self, datasource, form_data, slice_=None): def __init__(self, datasource, form_data, slice_=None):
self.orig_form_data = form_data
if not datasource: if not datasource:
raise Exception("Viz is missing a datasource") raise Exception("Viz is missing a datasource")
self.datasource = datasource self.datasource = datasource
@ -63,34 +62,6 @@ class BaseViz(object):
self.status = None self.status = None
self.error_message = None self.error_message = None
def get_filter_url(self):
"""Returns the URL to retrieve column values used in the filter"""
data = self.orig_form_data.copy()
# Remove unchecked checkboxes because HTML is weird like that
ordered_data = MultiDict()
for key in sorted(data.keys()):
# if MultiDict is initialized with MD({key:[emptyarray]}),
# key is included in d.keys() but accessing it throws
try:
if data[key] is False:
del data[key]
continue
except IndexError:
pass
if isinstance(data, (MultiDict, ImmutableMultiDict)):
v = data.getlist(key)
else:
v = data.get(key)
if not isinstance(v, list):
v = [v]
for item in v:
ordered_data.add(key, item)
href = Href(
'/superset/filter/{self.datasource.type}/'
'{self.datasource.id}/'.format(**locals()))
return href(ordered_data)
def get_df(self, query_obj=None): def get_df(self, query_obj=None):
"""Returns a pandas dataframe based on the query object""" """Returns a pandas dataframe based on the query object"""
if not query_obj: if not query_obj:
@ -277,7 +248,6 @@ class BaseViz(object):
'cache_timeout': cache_timeout, 'cache_timeout': cache_timeout,
'data': data, 'data': data,
'error': self.error_message, 'error': self.error_message,
'filter_endpoint': self.filter_endpoint,
'form_data': self.form_data, 'form_data': self.form_data,
'query': self.query, 'query': self.query,
'status': self.status, 'status': self.status,
@ -312,7 +282,6 @@ class BaseViz(object):
"""This is the data object serialized to the js layer""" """This is the data object serialized to the js layer"""
content = { content = {
'form_data': self.form_data, 'form_data': self.form_data,
'filter_endpoint': self.filter_endpoint,
'token': self.token, 'token': self.token,
'viz_name': self.viz_type, 'viz_name': self.viz_type,
'filter_select_enabled': self.datasource.filter_select_enabled, 'filter_select_enabled': self.datasource.filter_select_enabled,
@ -324,40 +293,9 @@ class BaseViz(object):
include_index = not isinstance(df.index, pd.RangeIndex) include_index = not isinstance(df.index, pd.RangeIndex)
return df.to_csv(index=include_index, encoding="utf-8") return df.to_csv(index=include_index, encoding="utf-8")
def get_values_for_column(self, column):
"""
Retrieves values for a column to be used by the filter dropdown.
:param column: column name
:return: JSON containing the some values for a column
"""
form_data = self.form_data
since = form_data.get("since", "1 year ago")
from_dttm = utils.parse_human_datetime(since)
now = datetime.now()
if from_dttm > now:
from_dttm = now - (from_dttm - now)
until = form_data.get("until", "now")
to_dttm = utils.parse_human_datetime(until)
if from_dttm > to_dttm:
raise Exception("From date cannot be larger than to date")
kwargs = dict(
column_name=column,
from_dttm=from_dttm,
to_dttm=to_dttm,
)
df = self.datasource.values_for_column(**kwargs)
return df[column].to_json()
def get_data(self, df): def get_data(self, df):
return [] return []
@property
def filter_endpoint(self):
return self.get_filter_url()
@property @property
def json_data(self): def json_data(self):
return json.dumps(self.data) return json.dumps(self.data)