[deck] allow an array of dynamic of aggregations (#6198)

* [deck] allow an array of dynamic of aggregations

* Adding quantiles

* lint & tests

superset/assets/package.json

@@ -62,6 +62,7 @@
     "brace": "^0.11.1",
     "classnames": "^2.2.5",
     "d3": "^3.5.17",
+    "d3-array": "^1.2.4",
     "d3-cloud": "^1.2.1",
     "d3-color": "^1.2.0",
     "d3-hierarchy": "^1.1.5",

superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx

@@ -0,0 +1,31 @@
+import { max } from 'd3-array';
+import { getAggFunc } from '../../../../../src/visualizations/deckgl/layers/common';
+
+describe('deckgl layers common', () => {
+  it('getAggFunc', () => {
+    const arr = [10, 0.5, 55, 128, -10];
+    expect(getAggFunc('max')(arr)).toEqual(128);
+    expect(getAggFunc('min')(arr)).toEqual(-10);
+    expect(getAggFunc('count')(arr)).toEqual(5);
+    expect(getAggFunc('median')(arr)).toEqual(10);
+    expect(getAggFunc('mean')(arr)).toEqual(36.7);
+    expect(getAggFunc('p1')(arr)).toEqual(-9.58);
+    expect(getAggFunc('p5')(arr)).toEqual(-7.9);
+    expect(getAggFunc('p95')(arr)).toEqual(113.39999999999998);
+    expect(getAggFunc('p99')(arr)).toEqual(125.08);
+  });
+  it('getAggFunc with accessor', () => {
+    const arr = [{ foo: 1 }, { foo: 2 }, { foo: 3 }];
+    const accessor = o => o.foo;
+    expect(getAggFunc('count')(arr, accessor)).toEqual(3);
+    expect(max(arr, accessor)).toEqual(3);
+    expect(getAggFunc('max', accessor)(arr)).toEqual(3);
+    expect(getAggFunc('min', accessor)(arr)).toEqual(1);
+    expect(getAggFunc('median', accessor)(arr)).toEqual(2);
+    expect(getAggFunc('mean', accessor)(arr)).toEqual(2);
+    expect(getAggFunc('p1', accessor)(arr)).toEqual(1.02);
+    expect(getAggFunc('p5', accessor)(arr)).toEqual(1.1);
+    expect(getAggFunc('p95', accessor)(arr)).toEqual(2.9);
+    expect(getAggFunc('p99', accessor)(arr)).toEqual(2.98);
+  });
+});

superset/assets/src/explore/controls.jsx

@@ -1317,6 +1317,29 @@ export const controls = {
     'computing the total rows and columns'),
   },
 
+  js_agg_function: {
+    type: 'SelectControl',
+    label: t('Dynamic Aggregation Function'),
+    description: t('The function to use when aggregating points into groups'),
+    default: 'sum',
+    clearable: false,
+    renderTrigger: true,
+    choices: formatSelectOptions([
+      'sum',
+      'min',
+      'max',
+      'mean',
+      'median',
+      'count',
+      'variance',
+      'deviation',
+      'p1',
+      'p5',
+      'p95',
+      'p99',
+    ]),
+  },
+
   size_from: {
     type: 'TextControl',
     isInt: true,

superset/assets/src/explore/visTypes.jsx

@@ -524,6 +524,7 @@ export const visTypes = {
           ['mapbox_style', 'viewport'],
           ['color_picker', 'autozoom'],
           ['grid_size', 'extruded'],
+          ['js_agg_function', null],
         ],
       },
       {

superset/assets/src/modules/sandbox.js

@@ -1,6 +1,7 @@
 // A safe alternative to JS's eval
 import vm from 'vm';
 import _ from 'underscore';
+import * as d3array from 'd3-array';
 import * as colors from './colors';
 
 // Objects exposed here should be treated like a public API
@@ -10,6 +11,7 @@ const GLOBAL_CONTEXT = {
   console,
   _,
   colors,
+  d3array,
 };
 
 // Copied/modified from https://github.com/hacksparrow/safe-eval/blob/master/index.js

superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx

@@ -1,5 +1,6 @@
 import { GridLayer } from 'deck.gl';
-import { commonLayerProps } from '../common';
+
+import { commonLayerProps, getAggFunc  } from '../common';
 import sandboxedEval from '../../../../modules/sandbox';
 import { createDeckGLComponent } from '../../factory';
 
@@ -17,6 +18,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
     data = jsFnMutator(data);
   }
 
+  const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
   return new GridLayer({
     id: `grid-layer-${fd.slice_id}`,
     data,
@@ -26,8 +28,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
     extruded: fd.extruded,
     maxColor: [c.r, c.g, c.b, 255 * c.a],
     outline: false,
-    getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+    getElevationValue: aggFunc,
-    getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+    getColorValue: aggFunc,
     ...commonLayerProps(fd, setTooltip),
   });
 }

superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx

@@ -1,5 +1,6 @@
 import { HexagonLayer } from 'deck.gl';
-import { commonLayerProps } from '../common';
+
+import { commonLayerProps, getAggFunc  } from '../common';
 import sandboxedEval from '../../../../modules/sandbox';
 import { createDeckGLComponent } from '../../factory';
 
@@ -16,7 +17,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
     const jsFnMutator = sandboxedEval(fd.js_data_mutator);
     data = jsFnMutator(data);
   }
-
+  const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
   return new HexagonLayer({
     id: `hex-layer-${fd.slice_id}`,
     data,
@@ -26,8 +27,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
     extruded: fd.extruded,
     maxColor: [c.r, c.g, c.b, 255 * c.a],
     outline: false,
-    getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+    getElevationValue: aggFunc,
-    getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+    getColorValue: aggFunc,
     ...commonLayerProps(fd, setTooltip),
   });
 }

superset/assets/src/visualizations/deckgl/layers/common.jsx

@@ -1,11 +1,12 @@
 import React from 'react';
+
 import { fitBounds } from 'viewport-mercator-project';
-import d3 from 'd3';
+import * as d3array from 'd3-array';
 import sandboxedEval from '../../../modules/sandbox';
 
 export function getBounds(points) {
-  const latExt = d3.extent(points, d => d[1]);
+  const latExt = d3array.extent(points, d => d[1]);
-  const lngExt = d3.extent(points, d => d[0]);
+  const lngExt = d3array.extent(points, d => d[0]);
   return [
     [lngExt[0], latExt[0]],
     [lngExt[1], latExt[1]],
@@ -73,3 +74,35 @@ export function commonLayerProps(formData, setTooltip, onSelect) {
     pickable: Boolean(onHover),
   };
 }
+
+const percentiles = {
+  p1: 0.01,
+  p5: 0.05,
+  p95: 0.95,
+  p99: 0.99,
+};
+
+/* Get an a stat function that operates on arrays, aligns with control=js_agg_function  */
+export function getAggFunc(type = 'sum', accessor = null) {
+  if (type === 'count') {
+    return arr => arr.length;
+  }
+  let d3func;
+  if (type in percentiles) {
+    d3func = (arr, acc) => {
+      let sortedArr;
+      if (accessor) {
+        sortedArr = arr.sort((o1, o2) => d3array.ascending(accessor(o1), accessor(o2)));
+      } else {
+        sortedArr = arr.sort(d3array.ascending);
+      }
+      return d3array.quantile(sortedArr, percentiles[type], acc);
+    };
+  } else {
+    d3func = d3array[type];
+  }
+  if (!accessor) {
+    return arr => d3func(arr);
+  }
+  return arr => d3func(arr.map(accessor));
+}

superset/assets/yarn.lock

@@ -3394,7 +3394,7 @@ cypress@^3.0.3:
     url "0.11.0"
     yauzl "2.8.0"
 
-d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1:
+d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1, d3-array@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"