feat: Typeahead searchable filter_box for dashboard (#10210)

* [WIP] Typeahead dashboard filter_box

* Make it work

* add config option for async filter_box

* enable for > 1000 options only

Co-authored-by: Jesse Yang <jesse.yang@airbnb.com>

superset-frontend/spec/javascripts/explore/components/FilterBoxItemControl_spec.jsx

@@ -50,7 +50,7 @@ describe('FilterBoxItemControl', () => {
 
   it('renderForms does the job', () => {
     const popover = shallow(inst.renderForm());
-    expect(popover.find(FormRow)).toHaveLength(7);
+    expect(popover.find(FormRow)).toHaveLength(8);
   });
 
   it('convert type for single value filter_box', () => {

superset-frontend/src/explore/components/controls/FilterBoxItemControl.jsx

@@ -53,6 +53,7 @@ const propTypes = {
   multiple: PropTypes.bool,
   column: PropTypes.string,
   metric: PropTypes.string,
+  searchAllOptions: PropTypes.bool,
   defaultValue: PropTypes.string,
 };
 
@@ -61,6 +62,7 @@ const defaultProps = {
   asc: true,
   clearable: true,
   multiple: true,
+  searchAllOptions: true,
 };
 
 const STYLE_WIDTH = { width: 350 };
@@ -68,8 +70,24 @@ const STYLE_WIDTH = { width: 350 };
 export default class FilterBoxItemControl extends React.Component {
   constructor(props) {
     super(props);
-    const { column, metric, asc, clearable, multiple, defaultValue } = props;
-    const state = { column, metric, asc, clearable, multiple, defaultValue };
+    const {
+      column,
+      metric,
+      asc,
+      clearable,
+      multiple,
+      searchAllOptions,
+      defaultValue,
+    } = props;
+    const state = {
+      column,
+      metric,
+      asc,
+      clearable,
+      multiple,
+      searchAllOptions,
+      defaultValue,
+    };
     this.state = state;
     this.onChange = this.onChange.bind(this);
     this.onControlChange = this.onControlChange.bind(this);
@@ -202,6 +220,26 @@ export default class FilterBoxItemControl extends React.Component {
             />
           }
         />
+        <FormRow
+          label={t('Search All Filter Options')}
+          tooltip={t(
+            'By default, each filter loads at most 1000 choices at the initial page load. ' +
+              'Check this box if you have more than 1000 filter values and want to enable dynamically ' +
+              'searching that loads filter values as users type (may add stress to your database).',
+          )}
+          isCheckbox
+          control={
+            <CheckboxControl
+              value={this.state.searchAllOptions}
+              onChange={v =>
+                this.onControlChange(
+                  FILTER_CONFIG_ATTRIBUTES.SEARCH_ALL_OPTIONS,
+                  v,
+                )
+              }
+            />
+          }
+        />
         <FormRow
           label={t('Required')}
           tooltip={t('User must select a value for this filter')}

superset-frontend/src/explore/constants.js

@@ -84,4 +84,8 @@ export const TIME_FILTER_LABELS = {
 export const FILTER_CONFIG_ATTRIBUTES = {
   DEFAULT_VALUE: 'defaultValue',
   MULTIPLE: 'multiple',
+  SEARCH_ALL_OPTIONS: 'searchAllOptions',
+  CLEARABLE: 'clearable',
 };
+
+export const FILTER_OPTIONS_LIMIT = 1000;

superset-frontend/src/visualizations/FilterBox/FilterBox.jsx

@@ -18,19 +18,24 @@
  */
 import React from 'react';
 import PropTypes from 'prop-types';
-import { CreatableSelect } from 'src/components/Select';
+import { debounce } from 'lodash';
+import { max as d3Max } from 'd3-array';
+import { AsyncCreatableSelect, CreatableSelect } from 'src/components/Select';
 import { Button } from 'react-bootstrap';
 import { t } from '@superset-ui/translation';
+import { SupersetClient } from '@superset-ui/connection';
 
 import DateFilterControl from '../../explore/components/controls/DateFilterControl';
 import ControlRow from '../../explore/components/ControlRow';
 import Control from '../../explore/components/Control';
 import controls from '../../explore/controls';
+import { getExploreUrl } from '../../explore/exploreUtils';
 import OnPasteSelect from '../../components/Select/OnPasteSelect';
 import { getDashboardFilterKey } from '../../dashboard/util/getDashboardFilterKey';
 import { getFilterColorMap } from '../../dashboard/util/dashboardFiltersColorMap';
 import {
   FILTER_CONFIG_ATTRIBUTES,
+  FILTER_OPTIONS_LIMIT,
   TIME_FILTER_LABELS,
 } from '../../explore/constants';
 import FilterBadgeIcon from '../../components/FilterBadgeIcon';
@@ -100,6 +105,8 @@ class FilterBox extends React.Component {
       // this flag is used by non-instant filter, to make the apply button enabled/disabled
       hasChanged: false,
     };
+    this.debouncerCache = {};
+    this.maxValueCache = {};
     this.changeFilter = this.changeFilter.bind(this);
     this.onFilterMenuOpen = this.onFilterMenuOpen.bind(this);
     this.onOpenDateFilterControl = this.onOpenDateFilterControl.bind(this);
@@ -131,13 +138,23 @@ class FilterBox extends React.Component {
     return mapFunc ? { ...control, ...mapFunc(this.props) } : control;
   }
 
+  /**
+   * Get known max value of a column
+   */
+  getKnownMax(key, choices) {
+    this.maxValueCache[key] = Math.max(
+      this.maxValueCache[key] || 0,
+      d3Max(choices || this.props.filtersChoices[key] || [], x => x.metric),
+    );
+    return this.maxValueCache[key];
+  }
+
   clickApply() {
     const { selectedValues } = this.state;
     this.setState({ hasChanged: false }, () => {
       this.props.onChange(selectedValues, false);
     });
   }
-
   changeFilter(filter, options) {
     const fltr = TIME_FILTER_MAP[filter] || filter;
     let vals = null;
@@ -162,6 +179,75 @@ class FilterBox extends React.Component {
     });
   }
 
+  /**
+   * Generate a debounce function that loads options for a specific column
+   */
+  debounceLoadOptions(key) {
+    if (!(key in this.debouncerCache)) {
+      this.debouncerCache[key] = debounce((input, callback) => {
+        this.loadOptions(key, input).then(callback);
+      }, 500);
+    }
+    return this.debouncerCache[key];
+  }
+
+  /**
+   * Transform select options, add bar background
+   */
+  transformOptions(options, max) {
+    const maxValue = max === undefined ? d3Max(options, x => x.metric) : max;
+    return options.map(opt => {
+      const perc = Math.round((opt.metric / maxValue) * 100);
+      const color = 'lightgrey';
+      const backgroundImage = `linear-gradient(to right, ${color}, ${color} ${perc}%, rgba(0,0,0,0) ${perc}%`;
+      const style = { backgroundImage };
+      return { value: opt.id, label: opt.id, style };
+    });
+  }
+
+  async loadOptions(key, inputValue = '') {
+    const input = inputValue.toLowerCase();
+    const sortAsc = this.props.filtersFields.find(x => x.key === key).asc;
+    const formData = {
+      ...this.props.rawFormData,
+      adhoc_filters: inputValue
+        ? [
+            {
+              clause: 'WHERE',
+              comparator: null,
+              expressionType: 'SQL',
+              // TODO: Evaluate SQL Injection risk
+              sqlExpression: `lower(${key}) like '%${input}%'`,
+            },
+          ]
+        : null,
+    };
+
+    const { json } = await SupersetClient.get({
+      url: getExploreUrl({
+        formData,
+        endpointType: 'json',
+        method: 'GET',
+      }),
+    });
+    const options = (json?.data?.[key] || []).filter(x => x.id);
+    if (!options || options.length === 0) {
+      return [];
+    }
+    if (input) {
+      // sort those starts with search query to front
+      options.sort((a, b) => {
+        const labelA = a.id.toLowerCase();
+        const labelB = b.id.toLowerCase();
+        const textOrder = labelB.startsWith(input) - labelA.startsWith(input);
+        return textOrder === 0
+          ? (a.metric - b.metric) * (sortAsc ? 1 : -1)
+          : textOrder;
+      });
+    }
+    return this.transformOptions(options, this.getKnownMax(key, options));
+  }
+
   renderDateFilter() {
     const { showDateFilter, chartId } = this.props;
     const label = TIME_FILTER_LABELS.time_range;
@@ -229,6 +315,8 @@ class FilterBox extends React.Component {
   renderSelect(filterConfig) {
     const { filtersChoices } = this.props;
     const { selectedValues } = this.state;
+    this.debouncerCache = {};
+    this.maxValueCache = {};
 
     // Add created options to filtersChoices, even though it doesn't exist,
     // or these options will exist in query sql but invisible to end user.
@@ -237,7 +325,7 @@ class FilterBox extends React.Component {
         key => selectedValues.hasOwnProperty(key) && key in filtersChoices,
       )
       .forEach(key => {
-        const choices = filtersChoices[key] || [];
+        const choices = filtersChoices[key] || (filtersChoices[key] = []);
         const choiceIds = new Set(choices.map(f => f.id));
         const selectedValuesForKey = Array.isArray(selectedValues[key])
           ? selectedValues[key]
@@ -255,7 +343,6 @@ class FilterBox extends React.Component {
       });
     const { key, label } = filterConfig;
     const data = filtersChoices[key] || [];
-    const max = Math.max(...data.map(d => d.metric));
     let value = selectedValues[key] || null;
 
     // Assign default value if required
@@ -273,20 +360,15 @@ class FilterBox extends React.Component {
 
     return (
       <OnPasteSelect
+        cacheOptions
+        loadOptions={this.debounceLoadOptions(key)}
+        defaultOptions={this.transformOptions(data)}
         key={key}
-        placeholder={t('Select [%s]', label)}
+        placeholder={t('Type or Select [%s]', label)}
         isMulti={filterConfig[FILTER_CONFIG_ATTRIBUTES.MULTIPLE]}
-        isClearable={filterConfig.clearable}
+        isClearable={filterConfig[FILTER_CONFIG_ATTRIBUTES.CLEARABLE]}
         value={value}
-        options={data
-          .filter(opt => opt.id !== null)
-          .map(opt => {
-            const perc = Math.round((opt.metric / max) * 100);
-            const color = 'lightgrey';
-            const backgroundImage = `linear-gradient(to right, ${color}, ${color} ${perc}%, rgba(0,0,0,0) ${perc}%`;
-            const style = { backgroundImage };
-            return { value: opt.id, label: opt.id, style };
-          })}
+        options={this.transformOptions(data)}
         onChange={newValue => {
           // avoid excessive re-renders
           if (newValue !== value) {
@@ -297,7 +379,12 @@ class FilterBox extends React.Component {
         onMenuOpen={() => this.onFilterMenuOpen(key)}
         onBlur={this.onFilterMenuClose}
         onMenuClose={this.onFilterMenuClose}
-        selectWrap={CreatableSelect}
+        selectWrap={
+          [FILTER_CONFIG_ATTRIBUTES.SEARCH_ALL_OPTIONS] &&
+          data.length >= FILTER_OPTIONS_LIMIT
+            ? AsyncCreatableSelect
+            : CreatableSelect
+        }
         noResultsText={t('No results found')}
       />
     );

superset-frontend/src/visualizations/FilterBox/transformProps.js

@@ -26,6 +26,7 @@ export default function transformProps(chartProps) {
     initialValues,
     queryData,
     rawDatasource,
+    rawFormData,
   } = chartProps;
   const {
     onAddFilter = NOOP,
@@ -65,5 +66,7 @@ export default function transformProps(chartProps) {
     showDruidTimeOrigin,
     showSqlaTimeColumn,
     showSqlaTimeGrain: showSqlaTimeGranularity,
+    // the original form data, needed for async select options
+    rawFormData,
   };
 }