mirror of https://github.com/apache/superset.git
[deck] allow an array of dynamic of aggregations (#6198)
* [deck] allow an array of dynamic of aggregations * Adding quantiles * lint & tests
This commit is contained in:
parent
7db11d95b0
commit
acb44165b4
|
@ -62,6 +62,7 @@
|
||||||
"brace": "^0.11.1",
|
"brace": "^0.11.1",
|
||||||
"classnames": "^2.2.5",
|
"classnames": "^2.2.5",
|
||||||
"d3": "^3.5.17",
|
"d3": "^3.5.17",
|
||||||
|
"d3-array": "^1.2.4",
|
||||||
"d3-cloud": "^1.2.1",
|
"d3-cloud": "^1.2.1",
|
||||||
"d3-color": "^1.2.0",
|
"d3-color": "^1.2.0",
|
||||||
"d3-hierarchy": "^1.1.5",
|
"d3-hierarchy": "^1.1.5",
|
||||||
|
|
|
@ -0,0 +1,31 @@
|
||||||
|
import { max } from 'd3-array';
|
||||||
|
import { getAggFunc } from '../../../../../src/visualizations/deckgl/layers/common';
|
||||||
|
|
||||||
|
describe('deckgl layers common', () => {
|
||||||
|
it('getAggFunc', () => {
|
||||||
|
const arr = [10, 0.5, 55, 128, -10];
|
||||||
|
expect(getAggFunc('max')(arr)).toEqual(128);
|
||||||
|
expect(getAggFunc('min')(arr)).toEqual(-10);
|
||||||
|
expect(getAggFunc('count')(arr)).toEqual(5);
|
||||||
|
expect(getAggFunc('median')(arr)).toEqual(10);
|
||||||
|
expect(getAggFunc('mean')(arr)).toEqual(36.7);
|
||||||
|
expect(getAggFunc('p1')(arr)).toEqual(-9.58);
|
||||||
|
expect(getAggFunc('p5')(arr)).toEqual(-7.9);
|
||||||
|
expect(getAggFunc('p95')(arr)).toEqual(113.39999999999998);
|
||||||
|
expect(getAggFunc('p99')(arr)).toEqual(125.08);
|
||||||
|
});
|
||||||
|
it('getAggFunc with accessor', () => {
|
||||||
|
const arr = [{ foo: 1 }, { foo: 2 }, { foo: 3 }];
|
||||||
|
const accessor = o => o.foo;
|
||||||
|
expect(getAggFunc('count')(arr, accessor)).toEqual(3);
|
||||||
|
expect(max(arr, accessor)).toEqual(3);
|
||||||
|
expect(getAggFunc('max', accessor)(arr)).toEqual(3);
|
||||||
|
expect(getAggFunc('min', accessor)(arr)).toEqual(1);
|
||||||
|
expect(getAggFunc('median', accessor)(arr)).toEqual(2);
|
||||||
|
expect(getAggFunc('mean', accessor)(arr)).toEqual(2);
|
||||||
|
expect(getAggFunc('p1', accessor)(arr)).toEqual(1.02);
|
||||||
|
expect(getAggFunc('p5', accessor)(arr)).toEqual(1.1);
|
||||||
|
expect(getAggFunc('p95', accessor)(arr)).toEqual(2.9);
|
||||||
|
expect(getAggFunc('p99', accessor)(arr)).toEqual(2.98);
|
||||||
|
});
|
||||||
|
});
|
|
@ -1317,6 +1317,29 @@ export const controls = {
|
||||||
'computing the total rows and columns'),
|
'computing the total rows and columns'),
|
||||||
},
|
},
|
||||||
|
|
||||||
|
js_agg_function: {
|
||||||
|
type: 'SelectControl',
|
||||||
|
label: t('Dynamic Aggregation Function'),
|
||||||
|
description: t('The function to use when aggregating points into groups'),
|
||||||
|
default: 'sum',
|
||||||
|
clearable: false,
|
||||||
|
renderTrigger: true,
|
||||||
|
choices: formatSelectOptions([
|
||||||
|
'sum',
|
||||||
|
'min',
|
||||||
|
'max',
|
||||||
|
'mean',
|
||||||
|
'median',
|
||||||
|
'count',
|
||||||
|
'variance',
|
||||||
|
'deviation',
|
||||||
|
'p1',
|
||||||
|
'p5',
|
||||||
|
'p95',
|
||||||
|
'p99',
|
||||||
|
]),
|
||||||
|
},
|
||||||
|
|
||||||
size_from: {
|
size_from: {
|
||||||
type: 'TextControl',
|
type: 'TextControl',
|
||||||
isInt: true,
|
isInt: true,
|
||||||
|
|
|
@ -524,6 +524,7 @@ export const visTypes = {
|
||||||
['mapbox_style', 'viewport'],
|
['mapbox_style', 'viewport'],
|
||||||
['color_picker', 'autozoom'],
|
['color_picker', 'autozoom'],
|
||||||
['grid_size', 'extruded'],
|
['grid_size', 'extruded'],
|
||||||
|
['js_agg_function', null],
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
// A safe alternative to JS's eval
|
// A safe alternative to JS's eval
|
||||||
import vm from 'vm';
|
import vm from 'vm';
|
||||||
import _ from 'underscore';
|
import _ from 'underscore';
|
||||||
|
import * as d3array from 'd3-array';
|
||||||
import * as colors from './colors';
|
import * as colors from './colors';
|
||||||
|
|
||||||
// Objects exposed here should be treated like a public API
|
// Objects exposed here should be treated like a public API
|
||||||
|
@ -10,6 +11,7 @@ const GLOBAL_CONTEXT = {
|
||||||
console,
|
console,
|
||||||
_,
|
_,
|
||||||
colors,
|
colors,
|
||||||
|
d3array,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Copied/modified from https://github.com/hacksparrow/safe-eval/blob/master/index.js
|
// Copied/modified from https://github.com/hacksparrow/safe-eval/blob/master/index.js
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import { GridLayer } from 'deck.gl';
|
import { GridLayer } from 'deck.gl';
|
||||||
import { commonLayerProps } from '../common';
|
|
||||||
|
import { commonLayerProps, getAggFunc } from '../common';
|
||||||
import sandboxedEval from '../../../../modules/sandbox';
|
import sandboxedEval from '../../../../modules/sandbox';
|
||||||
import { createDeckGLComponent } from '../../factory';
|
import { createDeckGLComponent } from '../../factory';
|
||||||
|
|
||||||
|
@ -17,6 +18,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
|
||||||
data = jsFnMutator(data);
|
data = jsFnMutator(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
|
||||||
return new GridLayer({
|
return new GridLayer({
|
||||||
id: `grid-layer-${fd.slice_id}`,
|
id: `grid-layer-${fd.slice_id}`,
|
||||||
data,
|
data,
|
||||||
|
@ -26,8 +28,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
|
||||||
extruded: fd.extruded,
|
extruded: fd.extruded,
|
||||||
maxColor: [c.r, c.g, c.b, 255 * c.a],
|
maxColor: [c.r, c.g, c.b, 255 * c.a],
|
||||||
outline: false,
|
outline: false,
|
||||||
getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
|
getElevationValue: aggFunc,
|
||||||
getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
|
getColorValue: aggFunc,
|
||||||
...commonLayerProps(fd, setTooltip),
|
...commonLayerProps(fd, setTooltip),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import { HexagonLayer } from 'deck.gl';
|
import { HexagonLayer } from 'deck.gl';
|
||||||
import { commonLayerProps } from '../common';
|
|
||||||
|
import { commonLayerProps, getAggFunc } from '../common';
|
||||||
import sandboxedEval from '../../../../modules/sandbox';
|
import sandboxedEval from '../../../../modules/sandbox';
|
||||||
import { createDeckGLComponent } from '../../factory';
|
import { createDeckGLComponent } from '../../factory';
|
||||||
|
|
||||||
|
@ -16,7 +17,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
|
||||||
const jsFnMutator = sandboxedEval(fd.js_data_mutator);
|
const jsFnMutator = sandboxedEval(fd.js_data_mutator);
|
||||||
data = jsFnMutator(data);
|
data = jsFnMutator(data);
|
||||||
}
|
}
|
||||||
|
const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
|
||||||
return new HexagonLayer({
|
return new HexagonLayer({
|
||||||
id: `hex-layer-${fd.slice_id}`,
|
id: `hex-layer-${fd.slice_id}`,
|
||||||
data,
|
data,
|
||||||
|
@ -26,8 +27,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) {
|
||||||
extruded: fd.extruded,
|
extruded: fd.extruded,
|
||||||
maxColor: [c.r, c.g, c.b, 255 * c.a],
|
maxColor: [c.r, c.g, c.b, 255 * c.a],
|
||||||
outline: false,
|
outline: false,
|
||||||
getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
|
getElevationValue: aggFunc,
|
||||||
getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
|
getColorValue: aggFunc,
|
||||||
...commonLayerProps(fd, setTooltip),
|
...commonLayerProps(fd, setTooltip),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
|
|
||||||
import { fitBounds } from 'viewport-mercator-project';
|
import { fitBounds } from 'viewport-mercator-project';
|
||||||
import d3 from 'd3';
|
import * as d3array from 'd3-array';
|
||||||
import sandboxedEval from '../../../modules/sandbox';
|
import sandboxedEval from '../../../modules/sandbox';
|
||||||
|
|
||||||
export function getBounds(points) {
|
export function getBounds(points) {
|
||||||
const latExt = d3.extent(points, d => d[1]);
|
const latExt = d3array.extent(points, d => d[1]);
|
||||||
const lngExt = d3.extent(points, d => d[0]);
|
const lngExt = d3array.extent(points, d => d[0]);
|
||||||
return [
|
return [
|
||||||
[lngExt[0], latExt[0]],
|
[lngExt[0], latExt[0]],
|
||||||
[lngExt[1], latExt[1]],
|
[lngExt[1], latExt[1]],
|
||||||
|
@ -73,3 +74,35 @@ export function commonLayerProps(formData, setTooltip, onSelect) {
|
||||||
pickable: Boolean(onHover),
|
pickable: Boolean(onHover),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const percentiles = {
|
||||||
|
p1: 0.01,
|
||||||
|
p5: 0.05,
|
||||||
|
p95: 0.95,
|
||||||
|
p99: 0.99,
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Get an a stat function that operates on arrays, aligns with control=js_agg_function */
|
||||||
|
export function getAggFunc(type = 'sum', accessor = null) {
|
||||||
|
if (type === 'count') {
|
||||||
|
return arr => arr.length;
|
||||||
|
}
|
||||||
|
let d3func;
|
||||||
|
if (type in percentiles) {
|
||||||
|
d3func = (arr, acc) => {
|
||||||
|
let sortedArr;
|
||||||
|
if (accessor) {
|
||||||
|
sortedArr = arr.sort((o1, o2) => d3array.ascending(accessor(o1), accessor(o2)));
|
||||||
|
} else {
|
||||||
|
sortedArr = arr.sort(d3array.ascending);
|
||||||
|
}
|
||||||
|
return d3array.quantile(sortedArr, percentiles[type], acc);
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
d3func = d3array[type];
|
||||||
|
}
|
||||||
|
if (!accessor) {
|
||||||
|
return arr => d3func(arr);
|
||||||
|
}
|
||||||
|
return arr => d3func(arr.map(accessor));
|
||||||
|
}
|
||||||
|
|
|
@ -3394,7 +3394,7 @@ cypress@^3.0.3:
|
||||||
url "0.11.0"
|
url "0.11.0"
|
||||||
yauzl "2.8.0"
|
yauzl "2.8.0"
|
||||||
|
|
||||||
d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1:
|
d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1, d3-array@^1.2.4:
|
||||||
version "1.2.4"
|
version "1.2.4"
|
||||||
resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"
|
resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue