From acb44165b406e5a35da9c21c1b6127ed8aca0b7a Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Mon, 29 Oct 2018 11:08:22 -0700 Subject: [PATCH] [deck] allow an array of dynamic of aggregations (#6198) * [deck] allow an array of dynamic of aggregations * Adding quantiles * lint & tests --- superset/assets/package.json | 1 + .../deckgl/layers/common_spec.jsx | 31 +++++++++++++++ superset/assets/src/explore/controls.jsx | 23 +++++++++++ superset/assets/src/explore/visTypes.jsx | 1 + superset/assets/src/modules/sandbox.js | 2 + .../deckgl/layers/Grid/Grid.jsx | 8 ++-- .../visualizations/deckgl/layers/Hex/Hex.jsx | 9 +++-- .../visualizations/deckgl/layers/common.jsx | 39 +++++++++++++++++-- superset/assets/yarn.lock | 2 +- 9 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx diff --git a/superset/assets/package.json b/superset/assets/package.json index f52096d34a..8293e96e45 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -62,6 +62,7 @@ "brace": "^0.11.1", "classnames": "^2.2.5", "d3": "^3.5.17", + "d3-array": "^1.2.4", "d3-cloud": "^1.2.1", "d3-color": "^1.2.0", "d3-hierarchy": "^1.1.5", diff --git a/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx b/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx new file mode 100644 index 0000000000..c11b994c0d --- /dev/null +++ b/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx @@ -0,0 +1,31 @@ +import { max } from 'd3-array'; +import { getAggFunc } from '../../../../../src/visualizations/deckgl/layers/common'; + +describe('deckgl layers common', () => { + it('getAggFunc', () => { + const arr = [10, 0.5, 55, 128, -10]; + expect(getAggFunc('max')(arr)).toEqual(128); + expect(getAggFunc('min')(arr)).toEqual(-10); + expect(getAggFunc('count')(arr)).toEqual(5); + expect(getAggFunc('median')(arr)).toEqual(10); + expect(getAggFunc('mean')(arr)).toEqual(36.7); + expect(getAggFunc('p1')(arr)).toEqual(-9.58); + expect(getAggFunc('p5')(arr)).toEqual(-7.9); + expect(getAggFunc('p95')(arr)).toEqual(113.39999999999998); + expect(getAggFunc('p99')(arr)).toEqual(125.08); + }); + it('getAggFunc with accessor', () => { + const arr = [{ foo: 1 }, { foo: 2 }, { foo: 3 }]; + const accessor = o => o.foo; + expect(getAggFunc('count')(arr, accessor)).toEqual(3); + expect(max(arr, accessor)).toEqual(3); + expect(getAggFunc('max', accessor)(arr)).toEqual(3); + expect(getAggFunc('min', accessor)(arr)).toEqual(1); + expect(getAggFunc('median', accessor)(arr)).toEqual(2); + expect(getAggFunc('mean', accessor)(arr)).toEqual(2); + expect(getAggFunc('p1', accessor)(arr)).toEqual(1.02); + expect(getAggFunc('p5', accessor)(arr)).toEqual(1.1); + expect(getAggFunc('p95', accessor)(arr)).toEqual(2.9); + expect(getAggFunc('p99', accessor)(arr)).toEqual(2.98); + }); +}); diff --git a/superset/assets/src/explore/controls.jsx b/superset/assets/src/explore/controls.jsx index d3a25a54b2..ad32c52dd3 100644 --- a/superset/assets/src/explore/controls.jsx +++ b/superset/assets/src/explore/controls.jsx @@ -1317,6 +1317,29 @@ export const controls = { 'computing the total rows and columns'), }, + js_agg_function: { + type: 'SelectControl', + label: t('Dynamic Aggregation Function'), + description: t('The function to use when aggregating points into groups'), + default: 'sum', + clearable: false, + renderTrigger: true, + choices: formatSelectOptions([ + 'sum', + 'min', + 'max', + 'mean', + 'median', + 'count', + 'variance', + 'deviation', + 'p1', + 'p5', + 'p95', + 'p99', + ]), + }, + size_from: { type: 'TextControl', isInt: true, diff --git a/superset/assets/src/explore/visTypes.jsx b/superset/assets/src/explore/visTypes.jsx index 97ac501e83..30aba88389 100644 --- a/superset/assets/src/explore/visTypes.jsx +++ b/superset/assets/src/explore/visTypes.jsx @@ -524,6 +524,7 @@ export const visTypes = { ['mapbox_style', 'viewport'], ['color_picker', 'autozoom'], ['grid_size', 'extruded'], + ['js_agg_function', null], ], }, { diff --git a/superset/assets/src/modules/sandbox.js b/superset/assets/src/modules/sandbox.js index a139013d59..1d50e1584c 100644 --- a/superset/assets/src/modules/sandbox.js +++ b/superset/assets/src/modules/sandbox.js @@ -1,6 +1,7 @@ // A safe alternative to JS's eval import vm from 'vm'; import _ from 'underscore'; +import * as d3array from 'd3-array'; import * as colors from './colors'; // Objects exposed here should be treated like a public API @@ -10,6 +11,7 @@ const GLOBAL_CONTEXT = { console, _, colors, + d3array, }; // Copied/modified from https://github.com/hacksparrow/safe-eval/blob/master/index.js diff --git a/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx b/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx index c9140aca24..b2ccd5bfaa 100644 --- a/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx +++ b/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx @@ -1,5 +1,6 @@ import { GridLayer } from 'deck.gl'; -import { commonLayerProps } from '../common'; + +import { commonLayerProps, getAggFunc } from '../common'; import sandboxedEval from '../../../../modules/sandbox'; import { createDeckGLComponent } from '../../factory'; @@ -17,6 +18,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { data = jsFnMutator(data); } + const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight); return new GridLayer({ id: `grid-layer-${fd.slice_id}`, data, @@ -26,8 +28,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { extruded: fd.extruded, maxColor: [c.r, c.g, c.b, 255 * c.a], outline: false, - getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0), - getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0), + getElevationValue: aggFunc, + getColorValue: aggFunc, ...commonLayerProps(fd, setTooltip), }); } diff --git a/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx b/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx index c3362eda2c..0e83782ee2 100644 --- a/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx +++ b/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx @@ -1,5 +1,6 @@ import { HexagonLayer } from 'deck.gl'; -import { commonLayerProps } from '../common'; + +import { commonLayerProps, getAggFunc } from '../common'; import sandboxedEval from '../../../../modules/sandbox'; import { createDeckGLComponent } from '../../factory'; @@ -16,7 +17,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { const jsFnMutator = sandboxedEval(fd.js_data_mutator); data = jsFnMutator(data); } - + const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight); return new HexagonLayer({ id: `hex-layer-${fd.slice_id}`, data, @@ -26,8 +27,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { extruded: fd.extruded, maxColor: [c.r, c.g, c.b, 255 * c.a], outline: false, - getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0), - getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0), + getElevationValue: aggFunc, + getColorValue: aggFunc, ...commonLayerProps(fd, setTooltip), }); } diff --git a/superset/assets/src/visualizations/deckgl/layers/common.jsx b/superset/assets/src/visualizations/deckgl/layers/common.jsx index a512526865..36b05cefa9 100644 --- a/superset/assets/src/visualizations/deckgl/layers/common.jsx +++ b/superset/assets/src/visualizations/deckgl/layers/common.jsx @@ -1,11 +1,12 @@ import React from 'react'; + import { fitBounds } from 'viewport-mercator-project'; -import d3 from 'd3'; +import * as d3array from 'd3-array'; import sandboxedEval from '../../../modules/sandbox'; export function getBounds(points) { - const latExt = d3.extent(points, d => d[1]); - const lngExt = d3.extent(points, d => d[0]); + const latExt = d3array.extent(points, d => d[1]); + const lngExt = d3array.extent(points, d => d[0]); return [ [lngExt[0], latExt[0]], [lngExt[1], latExt[1]], @@ -73,3 +74,35 @@ export function commonLayerProps(formData, setTooltip, onSelect) { pickable: Boolean(onHover), }; } + +const percentiles = { + p1: 0.01, + p5: 0.05, + p95: 0.95, + p99: 0.99, +}; + +/* Get an a stat function that operates on arrays, aligns with control=js_agg_function */ +export function getAggFunc(type = 'sum', accessor = null) { + if (type === 'count') { + return arr => arr.length; + } + let d3func; + if (type in percentiles) { + d3func = (arr, acc) => { + let sortedArr; + if (accessor) { + sortedArr = arr.sort((o1, o2) => d3array.ascending(accessor(o1), accessor(o2))); + } else { + sortedArr = arr.sort(d3array.ascending); + } + return d3array.quantile(sortedArr, percentiles[type], acc); + }; + } else { + d3func = d3array[type]; + } + if (!accessor) { + return arr => d3func(arr); + } + return arr => d3func(arr.map(accessor)); +} diff --git a/superset/assets/yarn.lock b/superset/assets/yarn.lock index 532d9a0cdf..a6ffc95c29 100644 --- a/superset/assets/yarn.lock +++ b/superset/assets/yarn.lock @@ -3394,7 +3394,7 @@ cypress@^3.0.3: url "0.11.0" yauzl "2.8.0" -d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1: +d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1, d3-array@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"