Enable freeform-select with fetched column values for filter values (#1697)

* Enable freeform-select with fetched column values for filter values
 - db migration to add filter_select_enabled
 - add freeform-multi option for Selectfield
 - modify formatFilter() function on query to accomodate filter-select

* Fix js tests

* Fix codeclimate issue

* Changes based on comments

* Add test for filter endpoint

* Extract out renderFilterFormField function from render

* Fix landscape issues

superset/assets/javascripts/explorev2/actions/exploreActions.js

@@ -93,6 +93,26 @@ export function changeFilter(filter, field, value) {
   return { type: CHANGE_FILTER, filter, field, value };
 }
 
+export function fetchFilterValues(datasource_type, datasource_id, filter, col) {
+  return function (dispatch) {
+    $.ajax({
+      type: 'GET',
+      url: `/superset/filter/${datasource_type}/${datasource_id}/${col}/`,
+      success: (data) => {
+        dispatch(changeFilter(
+          filter,
+          'choices',
+          Object.keys(data).map((k) => ([`'${data[k]}'`, `'${data[k]}'`]))
+          )
+        );
+      },
+      error() {
+        dispatch(changeFilter(filter, 'choices', []));
+      },
+    });
+  };
+}
+
 export const SET_FIELD_VALUE = 'SET_FIELD_VALUE';
 export function setFieldValue(datasource_type, key, value, label) {
   return { type: SET_FIELD_VALUE, datasource_type, key, value, label };

superset/assets/javascripts/explorev2/components/ControlPanelsContainer.jsx

@@ -102,6 +102,7 @@ class ControlPanelsContainer extends React.Component {
                 filters={this.props.form_data.filters}
                 actions={this.props.actions}
                 prefix={section.prefix}
+                datasource_id={this.props.form_data.datasource}
               />
             </ControlPanelSection>
           ))}

superset/assets/javascripts/explorev2/components/Filter.jsx

@@ -1,13 +1,16 @@
 import React from 'react';
-// import { Tab, Row, Col, Nav, NavItem } from 'react-bootstrap';
 import Select from 'react-select';
 import { Button } from 'react-bootstrap';
+import SelectField from './SelectField';
 
 const propTypes = {
   actions: React.PropTypes.object.isRequired,
   filterColumnOpts: React.PropTypes.array,
   prefix: React.PropTypes.string,
   filter: React.PropTypes.object.isRequired,
+  renderFilterSelect: React.PropTypes.bool,
+  datasource_type: React.PropTypes.string.isRequired,
+  datasource_id: React.PropTypes.number.isRequired,
 };
 
 const defaultProps = {
@@ -24,9 +27,22 @@ export default class Filter extends React.Component {
       opChoices,
     };
   }
+  componentWillMount() {
+    if (this.props.filter.col) {
+      this.props.actions.fetchFilterValues(
+        this.props.datasource_type,
+        this.props.datasource_id,
+        this.props.filter,
+        this.props.filter.col);
+    }
+  }
   changeCol(filter, colOpt) {
     const val = (colOpt) ? colOpt.value : null;
     this.props.actions.changeFilter(filter, 'col', val);
+    if (val) {
+      this.props.actions.fetchFilterValues(
+        this.props.datasource_type, this.props.datasource_id, filter, val);
+    }
   }
   changeOp(filter, opOpt) {
     const val = (opOpt) ? opOpt.value : null;
@@ -35,9 +51,35 @@ export default class Filter extends React.Component {
   changeValue(filter, event) {
     this.props.actions.changeFilter(filter, 'value', event.target.value);
   }
+  changeSelectValue(filter, name, value) {
+    this.props.actions.changeFilter(filter, 'value', value);
+  }
   removeFilter(filter) {
     this.props.actions.removeFilter(filter);
   }
+  renderFilterFormField() {
+    if (this.props.renderFilterSelect) {
+      return (
+        <SelectField
+          multi
+          freeForm
+          name="filter-value"
+          value={this.props.filter.value}
+          choices={this.props.filter.choices ? this.props.filter.choices : []}
+          onChange={this.changeSelectValue.bind(this, this.props.filter)}
+        />
+      );
+    }
+    return (
+      <input
+        type="text"
+        onChange={this.changeValue.bind(this, this.props.filter)}
+        value={this.props.filter.value}
+        className="form-control input-sm"
+        placeholder="Filter value"
+      />
+    );
+  }
   render() {
     return (
       <div>
@@ -65,13 +107,7 @@ export default class Filter extends React.Component {
             onChange={this.changeOp.bind(this, this.props.filter)}
           />
           <div className="col-lg-6">
-            <input
+            {this.renderFilterFormField()}
-              type="text"
-              onChange={this.changeValue.bind(this, this.props.filter)}
-              value={this.props.filter.value}
-              className="form-control input-sm"
-              placeholder="Filter value"
-            />
           </div>
           <div className="col-lg-2">
             <Button

superset/assets/javascripts/explorev2/components/Filters.jsx

@@ -7,9 +7,12 @@ import shortid from 'shortid';
 
 const propTypes = {
   actions: React.PropTypes.object.isRequired,
+  datasource_type: React.PropTypes.string.isRequired,
+  datasource_id: React.PropTypes.number.isRequired,
   filterColumnOpts: React.PropTypes.array,
   filters: React.PropTypes.array,
   prefix: React.PropTypes.string,
+  renderFilterSelect: React.PropTypes.bool,
 };
 
 const defaultProps = {
@@ -42,6 +45,9 @@ class Filters extends React.Component {
             actions={this.props.actions}
             prefix={this.props.prefix}
             filter={filter}
+            renderFilterSelect={this.props.renderFilterSelect}
+            datasource_type={this.props.datasource_type}
+            datasource_id={this.props.datasource_id}
           />
         );
       }
@@ -70,8 +76,10 @@ Filters.defaultProps = defaultProps;
 
 function mapStateToProps(state) {
   return {
+    datasource_type: state.datasource_type,
     filterColumnOpts: state.filterColumnOpts,
     filters: state.viz.form_data.filters,
+    renderFilterSelect: state.filter_select,
   };
 }
 

superset/assets/javascripts/explorev2/components/SelectField.jsx

@@ -58,10 +58,20 @@ export default class SelectField extends React.Component {
     if (this.props.freeForm) {
       // For FreeFormSelect, insert value into options if not exist
       const values = choices.map((c) => c[0]);
+      if (this.props.value) {
+        if (typeof this.props.value === 'object') {
+          this.props.value.forEach((v) => {
+            if (values.indexOf(v) === -1) {
+              options.push({ value: v, label: v });
+            }
+          });
+        } else {
           if (values.indexOf(this.props.value) === -1) {
             options.push({ value: this.props.value, label: this.props.value });
           }
         }
+      }
+    }
 
     const selectProps = {
       multi: this.props.multi,
@@ -77,7 +87,7 @@ export default class SelectField extends React.Component {
     //  Tab, comma or Enter will trigger a new option created for FreeFormSelect
     const selectWrap = this.props.freeForm ?
       (<Creatable {...selectProps} />) : (<Select {...selectProps} />);
-
+    if (this.props.label) {
       return (
         <div id={`formControlsSelect-${slugify(this.props.label)}`}>
           <ControlLabelWithTooltip
@@ -88,6 +98,12 @@ export default class SelectField extends React.Component {
         </div>
       );
     }
+    return (
+      <div>
+        {selectWrap}
+      </div>
+    );
+  }
 }
 
 SelectField.propTypes = propTypes;

superset/assets/javascripts/explorev2/index.jsx

@@ -25,6 +25,7 @@ const bootstrappedState = Object.assign(
   initialState(bootstrapData.viz.form_data.viz_type, bootstrapData.datasource_type), {
     can_edit: bootstrapData.can_edit,
     can_download: bootstrapData.can_download,
+    filter_select: bootstrapData.filter_select,
     datasources: bootstrapData.datasources,
     datasource_type: bootstrapData.datasource_type,
     viz: bootstrapData.viz,

superset/assets/javascripts/explorev2/stores/store.js

@@ -43,6 +43,7 @@ export function initialState(vizType = 'table', datasourceType = 'table') {
     datasources: null,
     datasource_type: null,
     filterColumnOpts: [],
+    filter_select: false,
     fields,
     viz: defaultViz(vizType, datasourceType),
     isStarred: false,

superset/assets/spec/javascripts/explorev2/components/Filter_spec.js

@@ -8,7 +8,9 @@ import { shallow } from 'enzyme';
 import Filter from '../../../../javascripts/explorev2/components/Filter';
 
 const defaultProps = {
-  actions: {},
+  actions: {
+    fetchFilterValues: () => ({}),
+  },
   filterColumnOpts: ['country_name'],
   filter: {
     id: 1,

superset/migrations/versions/f1f2d4af5b90_.py

@@ -0,0 +1,25 @@
+"""Enable Filter Select
+
+Revision ID: f1f2d4af5b90
+Revises: e46f2d27a08e
+Create Date: 2016-11-23 10:27:18.517919
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f1f2d4af5b90'
+down_revision = 'e46f2d27a08e'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('datasources', sa.Column('filter_select_enabled',
+                                           sa.Boolean(), default=False))
+    op.add_column('tables', sa.Column('filter_select_enabled',
+                                      sa.Boolean(), default=False))
+
+def downgrade():
+    op.drop_column('tables', 'filter_select_enabled')
+    op.drop_column('datasources', 'filter_select_enabled')

superset/models.py

@@ -32,6 +32,7 @@ from flask_appbuilder.models.decorators import renders
 from flask_babel import lazy_gettext as _
 
 from pydruid.client import PyDruid
+from pydruid.utils.aggregators import count
 from pydruid.utils.filters import Dimension, Filter
 from pydruid.utils.postaggregator import Postaggregator
 from pydruid.utils.having import Aggregation
@@ -866,6 +867,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable, ImportMixin):
     default_endpoint = Column(Text)
     database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
     is_featured = Column(Boolean, default=False)
+    filter_select_enabled = Column(Boolean, default=False)
     user_id = Column(Integer, ForeignKey('ab_user.id'))
     owner = relationship('User', backref='tables', foreign_keys=[user_id])
     database = relationship(
@@ -977,6 +979,45 @@ class SqlaTable(Model, Queryable, AuditMixinNullable, ImportMixin):
             if col_name == col.column_name:
                 return col
 
+    def values_for_column(self,
+                          column_name,
+                          from_dttm,
+                          to_dttm,
+                          limit=500):
+        """Runs query against sqla to retrieve some
+        sample values for the given column.
+        """
+        granularity = self.main_dttm_col
+
+        cols = {col.column_name: col for col in self.columns}
+        target_col = cols[column_name]
+
+        tbl = table(self.table_name)
+        qry = select([target_col.sqla_col])
+        qry = qry.select_from(tbl)
+        qry = qry.distinct(column_name)
+        qry = qry.limit(limit)
+
+        if granularity:
+            dttm_col = cols[granularity]
+            timestamp = dttm_col.sqla_col.label('timestamp')
+            time_filter = [
+                timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)),
+                timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)),
+            ]
+            qry = qry.where(and_(*time_filter))
+
+        engine = self.database.get_sqla_engine()
+        sql = "{}".format(
+            qry.compile(
+                engine, compile_kwargs={"literal_binds": True}, ),
+        )
+
+        return pd.read_sql_query(
+            sql=sql,
+            con=engine
+        )
+
     def query(  # sqla
             self, groupby, metrics,
             granularity,
@@ -1594,6 +1635,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
     datasource_name = Column(String(255), unique=True)
     is_featured = Column(Boolean, default=False)
     is_hidden = Column(Boolean, default=False)
+    filter_select_enabled = Column(Boolean, default=False)
     description = Column(Text)
     default_endpoint = Column(Text)
     user_id = Column(Integer, ForeignKey('ab_user.id'))
@@ -1930,6 +1972,35 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
                 period_name).total_seconds() * 1000
         return granularity
 
+    def values_for_column(self,
+                          column_name,
+                          from_dttm,
+                          to_dttm,
+                          limit=500):
+        """Retrieve some values for the given column"""
+        # TODO: Use Lexicographic TopNMeticSpec onces supported by PyDruid
+        from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+        to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+
+        qry = dict(
+            datasource=self.datasource_name,
+            granularity="all",
+            intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(),
+            aggregations=dict(count=count("count")),
+            dimension=column_name,
+            metric="count",
+            threshold=limit,
+        )
+
+        client = self.cluster.get_pydruid_client()
+        client.topn(**qry)
+        df = client.export_pandas()
+
+        if df is None or df.size == 0:
+            raise Exception(_("No data was returned."))
+
+        return df
+
     def query(  # druid
             self, groupby, metrics,
             granularity,

superset/views.py

@@ -646,7 +646,8 @@ class TableModelView(SupersetModelView, DeleteMixin):  # noqa
         'link', 'database', 'is_featured', 'changed_on_']
     add_columns = ['table_name', 'database', 'schema']
     edit_columns = [
-        'table_name', 'sql', 'is_featured', 'database', 'schema',
+        'table_name', 'sql', 'is_featured', 'filter_select_enabled',
+        'database', 'schema',
         'description', 'owner',
         'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
     show_columns = edit_columns + ['perm']
@@ -674,6 +675,7 @@ class TableModelView(SupersetModelView, DeleteMixin):  # noqa
         'database': _("Database"),
         'changed_on_': _("Last Changed"),
         'is_featured': _("Is Featured"),
+        'filter_select_enabled': _("Enable Filter Select"),
         'schema': _("Schema"),
         'default_endpoint': _("Default Endpoint"),
         'offset': _("Offset"),
@@ -1031,8 +1033,8 @@ class DruidDatasourceModelView(SupersetModelView, DeleteMixin):  # noqa
     related_views = [DruidColumnInlineView, DruidMetricInlineView]
     edit_columns = [
         'datasource_name', 'cluster', 'description', 'owner',
-        'is_featured', 'is_hidden', 'default_endpoint', 'offset',
+        'is_featured', 'is_hidden', 'filter_select_enabled',
-        'cache_timeout']
+        'default_endpoint', 'offset', 'cache_timeout']
     add_columns = edit_columns
     show_columns = add_columns + ['perm']
     page_size = 500
@@ -1051,6 +1053,7 @@ class DruidDatasourceModelView(SupersetModelView, DeleteMixin):  # noqa
         'owner': _("Owner"),
         'is_featured': _("Is Featured"),
         'is_hidden': _("Is Hidden"),
+        'filter_select_enabled': _("Enable Filter Select"),
         'default_endpoint': _("Default Endpoint"),
         'offset': _("Time Offset"),
         'cache_timeout': _("Cache Timeout"),
@@ -1494,7 +1497,8 @@ class Superset(BaseSupersetView):
                 "datasource_name": viz_obj.datasource.name,
                 "datasource_type": datasource_type,
                 "user_id": user_id,
-                "viz": json.loads(viz_obj.json_data)
+                "viz": json.loads(viz_obj.json_data),
+                "filter_select": viz_obj.datasource.filter_select_enabled
             }
             table_name = viz_obj.datasource.table_name \
                 if datasource_type == 'table' \
@@ -1513,6 +1517,53 @@ class Superset(BaseSupersetView):
                 userid=g.user.get_id() if g.user else ''
             )
 
+    @api
+    @has_access_api
+    @expose("/filter/<datasource_type>/<datasource_id>/<column>/")
+    def filter(self, datasource_type, datasource_id, column):
+        """
+        Endpoint to retrieve values for specified column.
+
+        :param datasource_type: Type of datasource e.g. table
+        :param datasource_id: Datasource id
+        :param column: Column name to retrieve values for
+        :return:
+        """
+        # TODO: Cache endpoint by user, datasource and column
+        error_redirect = '/slicemodelview/list/'
+        datasource_class = models.SqlaTable \
+            if datasource_type == "table" else models.DruidDatasource
+
+        datasource = db.session.query(
+            datasource_class).filter_by(id=datasource_id).first()
+
+        if not datasource:
+            flash(DATASOURCE_MISSING_ERR, "alert")
+            return json_error_response(DATASOURCE_MISSING_ERR)
+        if not self.datasource_access(datasource):
+            flash(get_datasource_access_error_msg(datasource.name), "danger")
+            return json_error_response(DATASOURCE_ACCESS_ERR)
+
+        viz_type = request.args.get("viz_type")
+        if not viz_type and datasource.default_endpoint:
+            return redirect(datasource.default_endpoint)
+        if not viz_type:
+            viz_type = "table"
+        try:
+            obj = viz.viz_types[viz_type](
+                datasource,
+                form_data=request.args,
+                slice_=None)
+        except Exception as e:
+            flash(str(e), "danger")
+            return redirect(error_redirect)
+        status = 200
+        payload = obj.get_values_for_column(column)
+        return Response(
+            payload,
+            status=status,
+            mimetype="application/json")
+
     def save_or_overwrite_slice(
             self, args, slc, slice_add_perm, slice_edit_perm):
         """Save or overwrite a slice"""

superset/viz.py

@@ -151,6 +151,34 @@ class BaseViz(object):
             del od['force']
         return href(od)
 
+    def get_filter_url(self):
+        """Returns the URL to retrieve column values used in the filter"""
+        data = self.orig_form_data.copy()
+        # Remove unchecked checkboxes because HTML is weird like that
+        ordered_data = MultiDict()
+        for key in sorted(data.keys()):
+            # if MultiDict is initialized with MD({key:[emptyarray]}),
+            # key is included in d.keys() but accessing it throws
+            try:
+                if data[key] is False:
+                    del data[key]
+                    continue
+            except IndexError:
+                pass
+
+            if isinstance(data, (MultiDict, ImmutableMultiDict)):
+                v = data.getlist(key)
+            else:
+                v = data.get(key)
+            if not isinstance(v, list):
+                v = [v]
+            for item in v:
+                ordered_data.add(key, item)
+        href = Href(
+            '/caravel/filter/{self.datasource.type}/'
+            '{self.datasource.id}/'.format(**locals()))
+        return href(ordered_data)
+
     def get_df(self, query_obj=None):
         """Returns a pandas dataframe based on the query object"""
         if not query_obj:
@@ -325,6 +353,7 @@ class BaseViz(object):
                 'form_data': self.form_data,
                 'json_endpoint': self.json_endpoint,
                 'query': self.query,
+                'filter_endpoint': self.filter_endpoint,
                 'standalone_endpoint': self.standalone_endpoint,
                 'column_formats': self.data['column_formats'],
             }
@@ -359,9 +388,11 @@ class BaseViz(object):
             'csv_endpoint': self.csv_endpoint,
             'form_data': self.form_data,
             'json_endpoint': self.json_endpoint,
+            'filter_endpoint': self.filter_endpoint,
             'standalone_endpoint': self.standalone_endpoint,
             'token': self.token,
             'viz_name': self.viz_type,
+            'filter_select_enabled': self.datasource.filter_select_enabled,
             'column_formats': {
                 m.metric_name: m.d3format
                 for m in self.datasource.metrics
@@ -375,6 +406,34 @@ class BaseViz(object):
         include_index = not isinstance(df.index, pd.RangeIndex)
         return df.to_csv(index=include_index, encoding="utf-8")
 
+    def get_values_for_column(self, column):
+        """
+        Retrieves values for a column to be used by the filter dropdown.
+
+        :param column: column name
+        :return: JSON containing the some values for a column
+        """
+        form_data = self.form_data
+
+        since = form_data.get("since", "1 year ago")
+        from_dttm = utils.parse_human_datetime(since)
+        now = datetime.now()
+        if from_dttm > now:
+            from_dttm = now - (from_dttm - now)
+        until = form_data.get("until", "now")
+        to_dttm = utils.parse_human_datetime(until)
+        if from_dttm > to_dttm:
+            flasher("The date range doesn't seem right.", "danger")
+            from_dttm = to_dttm  # Making them identical to not raise
+
+        kwargs = dict(
+            column_name=column,
+            from_dttm=from_dttm,
+            to_dttm=to_dttm,
+        )
+        df = self.datasource.values_for_column(**kwargs)
+        return df[column].to_json()
+
     def get_data(self):
         return []
 
@@ -382,6 +441,10 @@ class BaseViz(object):
     def json_endpoint(self):
         return self.get_url(json_endpoint=True)
 
+    @property
+    def filter_endpoint(self):
+        return self.get_filter_url()
+
     @property
     def cache_key(self):
         url = self.get_url(for_cache_key=True, json="true", force="false")

tests/core_tests.py

@@ -122,6 +122,25 @@ class CoreTests(SupersetTestCase):
         assert 'Energy' in self.get_resp(
             url.format(tbl_id, slice_id, copy_name, 'overwrite'))
 
+    def test_filter_endpoint(self):
+        self.login(username='admin')
+        slice_name = "Energy Sankey"
+        slice_id = self.get_slice(slice_name, db.session).id
+        db.session.commit()
+        tbl_id = self.table_ids.get('energy_usage')
+        table = db.session.query(models.SqlaTable).filter(models.SqlaTable.id == tbl_id)
+        table.filter_select_enabled = True
+        url = (
+            "/superset/filter/table/{}/target/?viz_type=sankey&groupby=source"
+            "&metric=sum__value&flt_col_0=source&flt_op_0=in&flt_eq_0=&"
+            "slice_id={}&datasource_name=energy_usage&"
+            "datasource_id=1&datasource_type=table")
+
+        # Changing name
+        resp = self.get_resp(url.format(tbl_id, slice_id))
+        assert len(resp) > 0
+        assert 'Carbon Dioxide' in resp
+
     def test_slices(self):
         # Testing by hitting the two supported end points for all slices
         self.login(username='admin')