Multi layers DECK.GL visualization (#4096)

* Multi layers DECK.GL viz

* Fix tests

* rebasing

* Fix error handling in chartActions

* Addressing comments

setup.py

@@ -73,7 +73,7 @@ setup(
         'pyyaml>=3.11',
         'requests==2.17.3',
         'simplejson==3.10.0',
-        'six==1.10.0',
+        'six==1.11.0',
         'sqlalchemy==1.1.9',
         'sqlalchemy-utils==0.32.16',
         'sqlparse==0.2.3',

superset/assets/javascripts/chart/chartAction.js

@@ -120,7 +120,20 @@ export function runQuery(formData, force = false, timeout = 60, key) {
         if (err.statusText === 'timeout') {
           dispatch(chartUpdateTimeout(err.statusText, timeout, key));
         } else if (err.statusText !== 'abort') {
-          dispatch(chartUpdateFailed(err.responseJSON, key));
+          let errObject;
+          if (err.responseJSON) {
+            errObject = err.responseJSON;
+          } else if (err.stack) {
+            errObject = {
+              error: 'Unexpected error: ' + err.description,
+              stacktrace: err.stack,
+            };
+          } else {
+            errObject = {
+              error: 'Unexpected error.',
+            };
+          }
+          dispatch(chartUpdateFailed(errObject, key));
         }
       });
     const annotationLayers = formData.annotation_layers || [];

superset/assets/javascripts/explore/stores/controls.jsx

@@ -1389,6 +1389,7 @@ export const controls = {
   mapbox_style: {
     type: 'SelectControl',
     label: t('Map Style'),
+    clearable: false,
     renderTrigger: true,
     choices: [
       ['mapbox://styles/mapbox/streets-v9', 'Streets'],
@@ -1816,5 +1817,23 @@ export const controls = {
         and returns a similarly shaped object. {sandboxedEvalInfo}
       </p>),
   },
+
+  deck_slices: {
+    type: 'SelectAsyncControl',
+    multi: true,
+    label: t('deck.gl charts'),
+    validators: [v.nonEmpty],
+    default: [],
+    description: t('Pick a set of deck.gl charts to layer on top of one another'),
+    dataEndpoint: '/sliceasync/api/read?_flt_0_viz_type=deck_',
+    placeholder: t('Select charts'),
+    onAsyncErrorMessage: t('Error while fetching charts'),
+    mutator: (data) => {
+      if (!data || !data.result) {
+        return [];
+      }
+      return data.result.map(o => ({ value: o.id, label: o.slice_name }));
+    },
+  },
 };
 export default controls;

superset/assets/javascripts/explore/stores/visTypes.js

@@ -338,6 +338,21 @@ export const visTypes = {
     },
   },
 
+  deck_multi: {
+    label: t('Deck.gl - Multiple Layers'),
+    requiresTime: true,
+    controlPanelSections: [
+      {
+        label: t('Map'),
+        expanded: true,
+        controlSetRows: [
+          ['mapbox_style', 'viewport'],
+          ['deck_slices', null],
+        ],
+      },
+    ],
+  },
+
   deck_hex: {
     label: t('Deck.gl - Hexagons'),
     requiresTime: true,
@@ -398,7 +413,7 @@ export const visTypes = {
   },
 
   deck_path: {
-    label: t('Deck.gl - Grid'),
+    label: t('Deck.gl - Paths'),
     requiresTime: true,
     controlPanelSections: [
       {

superset/assets/visualizations/deckgl/factory.jsx

@@ -1,25 +1,12 @@
 import React from 'react';
 import ReactDOM from 'react-dom';
-import { PathLayer } from 'deck.gl';
 
 import DeckGLContainer from './DeckGLContainer';
+import layerGenerators from './layers';
 
-function deckPath(slice, payload, setControlValue) {
+export default function deckglFactory(slice, payload, setControlValue) {
   const fd = slice.formData;
-  const c = fd.color_picker;
-  const fixedColor = [c.r, c.g, c.b, 255 * c.a];
-  const data = payload.data.paths.map(path => ({
-    path,
-    width: fd.line_width,
-    color: fixedColor,
-  }));
-
-  const layer = new PathLayer({
-    id: `path-layer-${slice.containerId}`,
-    data,
-    rounded: true,
-    widthScale: 1,
-  });
+  const layer = layerGenerators[fd.viz_type](fd, payload);
   const viewport = {
     ...fd.viewport,
     width: slice.width(),
@@ -36,4 +23,3 @@ function deckPath(slice, payload, setControlValue) {
     document.getElementById(slice.containerId),
   );
 }
-module.exports = deckPath;

superset/assets/visualizations/deckgl/grid.jsx

@@ -1,43 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom';
-import { GridLayer } from 'deck.gl';
-
-import DeckGLContainer from './DeckGLContainer';
-
-function deckScreenGridLayer(slice, payload, setControlValue) {
-  const fd = slice.formData;
-  const c = fd.color_picker;
-  const data = payload.data.features.map(d => ({
-    ...d,
-    color: [c.r, c.g, c.b, 255 * c.a],
-  }));
-
-  const layer = new GridLayer({
-    id: `grid-layer-${slice.containerId}`,
-    data,
-    pickable: true,
-    cellSize: fd.grid_size,
-    minColor: [0, 0, 0, 0],
-    extruded: fd.extruded,
-    maxColor: [c.r, c.g, c.b, 255 * c.a],
-    outline: false,
-    getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
-    getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
-  });
-  const viewport = {
-    ...fd.viewport,
-    width: slice.width(),
-    height: slice.height(),
-  };
-  ReactDOM.render(
-    <DeckGLContainer
-      mapboxApiAccessToken={payload.data.mapboxApiKey}
-      viewport={viewport}
-      layers={[layer]}
-      mapStyle={fd.mapbox_style}
-      setControlValue={setControlValue}
-    />,
-    document.getElementById(slice.containerId),
-  );
-}
-module.exports = deckScreenGridLayer;

superset/assets/visualizations/deckgl/hex.jsx

@@ -1,43 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom';
-import { HexagonLayer } from 'deck.gl';
-
-import DeckGLContainer from './DeckGLContainer';
-
-function deckHex(slice, payload, setControlValue) {
-  const fd = slice.formData;
-  const c = fd.color_picker;
-  const data = payload.data.features.map(d => ({
-    ...d,
-    color: [c.r, c.g, c.b, 255 * c.a],
-  }));
-
-  const layer = new HexagonLayer({
-    id: `hex-layer-${slice.containerId}`,
-    data,
-    pickable: true,
-    radius: fd.grid_size,
-    minColor: [0, 0, 0, 0],
-    extruded: fd.extruded,
-    maxColor: [c.r, c.g, c.b, 255 * c.a],
-    outline: false,
-    getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
-    getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
-  });
-  const viewport = {
-    ...fd.viewport,
-    width: slice.width(),
-    height: slice.height(),
-  };
-  ReactDOM.render(
-    <DeckGLContainer
-      mapboxApiAccessToken={payload.data.mapboxApiKey}
-      viewport={viewport}
-      layers={[layer]}
-      mapStyle={fd.mapbox_style}
-      setControlValue={setControlValue}
-    />,
-    document.getElementById(slice.containerId),
-  );
-}
-module.exports = deckHex;

superset/assets/visualizations/deckgl/layers/geojson.jsx

@@ -1,9 +1,6 @@
-import React from 'react';
-import ReactDOM from 'react-dom';
 import { GeoJsonLayer } from 'deck.gl';
-import { hexToRGB } from '../../javascripts/modules/colors';
+import { hexToRGB } from '../../../javascripts/modules/colors';
 
-import DeckGLContainer from './DeckGLContainer';
 
 const propertyMap = {
   fillColor: 'fillColor',
@@ -26,8 +23,8 @@ const convertGeoJsonColorProps = (p, colors) => {
   };
 };
 
-function DeckGeoJsonLayer(slice, payload, setControlValue) {
-  const fd = slice.formData;
+export default function geoJsonLayer(formData, payload) {
+  const fd = formData;
   const fc = fd.fill_color_picker;
   const sc = fd.stroke_color_picker;
   const data = payload.data.geojson.features.map(d => ({
@@ -39,29 +36,12 @@ function DeckGeoJsonLayer(slice, payload, setControlValue) {
       }),
   }));
 
-  const layer = new GeoJsonLayer({
-    id: 'geojson-layer',
+  return new GeoJsonLayer({
+    id: `path-layer-${fd.slice_id}`,
     data,
     filled: true,
     stroked: false,
     extruded: true,
     pointRadiusScale: fd.point_radius_scale,
   });
-
-  const viewport = {
-    ...fd.viewport,
-    width: slice.width(),
-    height: slice.height(),
-  };
-  ReactDOM.render(
-    <DeckGLContainer
-      mapboxApiAccessToken={payload.data.mapboxApiKey}
-      viewport={viewport}
-      layers={[layer]}
-      mapStyle={fd.mapbox_style}
-      setControlValue={setControlValue}
-    />,
-    document.getElementById(slice.containerId),
-  );
 }
-module.exports = DeckGeoJsonLayer;

superset/assets/visualizations/deckgl/layers/grid.jsx

@@ -0,0 +1,23 @@
+import { GridLayer } from 'deck.gl';
+
+export default function getLayer(formData, payload) {
+  const fd = formData;
+  const c = fd.color_picker;
+  const data = payload.data.features.map(d => ({
+    ...d,
+    color: [c.r, c.g, c.b, 255 * c.a],
+  }));
+
+  return new GridLayer({
+    id: `grid-layer-${fd.slice_id}`,
+    data,
+    pickable: true,
+    cellSize: fd.grid_size,
+    minColor: [0, 0, 0, 0],
+    extruded: fd.extruded,
+    maxColor: [c.r, c.g, c.b, 255 * c.a],
+    outline: false,
+    getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+    getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+  });
+}

superset/assets/visualizations/deckgl/layers/hex.jsx

@@ -0,0 +1,23 @@
+import { HexagonLayer } from 'deck.gl';
+
+export default function getLayer(formData, payload) {
+  const fd = formData;
+  const c = fd.color_picker;
+  const data = payload.data.features.map(d => ({
+    ...d,
+    color: [c.r, c.g, c.b, 255 * c.a],
+  }));
+
+  return new HexagonLayer({
+    id: `hex-layer-${fd.slice_id}`,
+    data,
+    pickable: true,
+    radius: fd.grid_size,
+    minColor: [0, 0, 0, 0],
+    extruded: fd.extruded,
+    maxColor: [c.r, c.g, c.b, 255 * c.a],
+    outline: false,
+    getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+    getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0),
+  });
+}

superset/assets/visualizations/deckgl/layers/index.js

@@ -0,0 +1,17 @@
+/* eslint camelcase: 0 */
+import deck_grid from './grid';
+import deck_screengrid from './screengrid';
+import deck_path from './path';
+import deck_hex from './hex';
+import deck_scatter from './scatter';
+import deck_geojson from './geojson';
+
+const layerGenerators = {
+  deck_grid,
+  deck_screengrid,
+  deck_path,
+  deck_hex,
+  deck_scatter,
+  deck_geojson,
+};
+export default layerGenerators;

superset/assets/visualizations/deckgl/layers/path.jsx

@@ -0,0 +1,19 @@
+import { PathLayer } from 'deck.gl';
+
+export default function getLayer(formData, payload) {
+  const fd = formData;
+  const c = fd.color_picker;
+  const fixedColor = [c.r, c.g, c.b, 255 * c.a];
+  const data = payload.data.paths.map(path => ({
+    path,
+    width: fd.line_width,
+    color: fixedColor,
+  }));
+
+  return new PathLayer({
+    id: `path-layer-${fd.slice_id}`,
+    data,
+    rounded: true,
+    widthScale: 1,
+  });
+}

superset/assets/visualizations/deckgl/layers/scatter.jsx

@@ -0,0 +1,35 @@
+import { ScatterplotLayer } from 'deck.gl';
+
+import { getColorFromScheme, hexToRGB } from '../../../javascripts/modules/colors';
+import { unitToRadius } from '../../../javascripts/modules/geo';
+
+export default function getLayer(formData, payload) {
+  const fd = formData;
+  const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 };
+  const fixedColor = [c.r, c.g, c.b, 255 * c.a];
+
+  const data = payload.data.features.map((d) => {
+    let radius = unitToRadius(fd.point_unit, d.radius) || 10;
+    if (fd.multiplier) {
+      radius *= fd.multiplier;
+    }
+    let color;
+    if (fd.dimension) {
+      color = hexToRGB(getColorFromScheme(d.cat_color, fd.color_scheme), c.a * 255);
+    } else {
+      color = fixedColor;
+    }
+    return {
+      ...d,
+      radius,
+      color,
+    };
+  });
+  return new ScatterplotLayer({
+    id: `scatter-layer-${fd.slice_id}`,
+    data,
+    pickable: true,
+    fp64: true,
+    outline: false,
+  });
+}

superset/assets/visualizations/deckgl/layers/screengrid.jsx

@@ -0,0 +1,23 @@
+import { ScreenGridLayer } from 'deck.gl';
+
+export default function getLayer(formData, payload) {
+  const fd = formData;
+  const c = fd.color_picker;
+  const data = payload.data.features.map(d => ({
+    ...d,
+    color: [c.r, c.g, c.b, 255 * c.a],
+  }));
+
+  // Passing a layer creator function instead of a layer since the
+  // layer needs to be regenerated at each render
+  return new ScreenGridLayer({
+    id: `screengrid-layer-${fd.slice_id}`,
+    data,
+    pickable: true,
+    cellSizePixels: fd.grid_size,
+    minColor: [c.r, c.g, c.b, 0],
+    maxColor: [c.r, c.g, c.b, 255 * c.a],
+    outline: false,
+    getWeight: d => d.weight || 0,
+  });
+}

superset/assets/visualizations/deckgl/multi.jsx

@@ -0,0 +1,44 @@
+import React from 'react';
+import ReactDOM from 'react-dom';
+import $ from 'jquery';
+
+import DeckGLContainer from './DeckGLContainer';
+import { getExploreUrl } from '../../javascripts/explore/exploreUtils';
+import layerGenerators from './layers';
+
+
+function deckMulti(slice, payload, setControlValue) {
+  if (!slice.subSlicesLayers) {
+    slice.subSlicesLayers = {}; // eslint-disable-line no-param-reassign
+  }
+  const fd = slice.formData;
+  const render = () => {
+    const viewport = {
+      ...fd.viewport,
+      width: slice.width(),
+      height: slice.height(),
+    };
+    const layers = Object.keys(slice.subSlicesLayers).map(k => slice.subSlicesLayers[k]);
+    ReactDOM.render(
+      <DeckGLContainer
+        mapboxApiAccessToken={payload.data.mapboxApiKey}
+        viewport={viewport}
+        layers={layers}
+        mapStyle={fd.mapbox_style}
+        setControlValue={setControlValue}
+      />,
+      document.getElementById(slice.containerId),
+    );
+  };
+  render();
+  payload.data.slices.forEach((subslice) => {
+    const url = getExploreUrl(subslice.form_data, 'json');
+    $.get(url, (data) => {
+      // Late import to avoid circular deps
+      const layer = layerGenerators[subslice.form_data.viz_type](subslice.form_data, data);
+      slice.subSlicesLayers[subslice.slice_id] = layer; // eslint-disable-line no-param-reassign
+      render();
+    });
+  });
+}
+module.exports = deckMulti;

superset/assets/visualizations/deckgl/scatter.jsx

@@ -1,55 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom';
-import { ScatterplotLayer } from 'deck.gl';
-
-import DeckGLContainer from './DeckGLContainer';
-import { getColorFromScheme, hexToRGB } from '../../javascripts/modules/colors';
-import { unitToRadius } from '../../javascripts/modules/geo';
-
-function deckScatter(slice, payload, setControlValue) {
-  const fd = slice.formData;
-  const c = fd.color_picker || { r: 0, g: 0, b: 0, a: 1 };
-  const fixedColor = [c.r, c.g, c.b, 255 * c.a];
-
-  const data = payload.data.features.map((d) => {
-    let radius = unitToRadius(fd.point_unit, d.radius) || 10;
-    if (fd.multiplier) {
-      radius *= fd.multiplier;
-    }
-    let color;
-    if (fd.dimension) {
-      color = hexToRGB(getColorFromScheme(d.cat_color, fd.color_scheme), c.a * 255);
-    } else {
-      color = fixedColor;
-    }
-    return {
-      ...d,
-      radius,
-      color,
-    };
-  });
-
-  const layer = new ScatterplotLayer({
-    id: `scatter-layer-${slice.containerId}`,
-    data,
-    pickable: true,
-    fp64: true,
-    outline: false,
-  });
-  const viewport = {
-    ...fd.viewport,
-    width: slice.width(),
-    height: slice.height(),
-  };
-  ReactDOM.render(
-    <DeckGLContainer
-      mapboxApiAccessToken={payload.data.mapboxApiKey}
-      viewport={viewport}
-      layers={[layer]}
-      mapStyle={fd.mapbox_style}
-      setControlValue={setControlValue}
-    />,
-    document.getElementById(slice.containerId),
-  );
-}
-module.exports = deckScatter;

superset/assets/visualizations/deckgl/screengrid.jsx

@@ -1,43 +0,0 @@
-import React from 'react';
-import ReactDOM from 'react-dom';
-import { ScreenGridLayer } from 'deck.gl';
-
-import DeckGLContainer from './DeckGLContainer';
-
-function deckScreenGridLayer(slice, payload, setControlValue) {
-  const fd = slice.formData;
-  const c = fd.color_picker;
-  const data = payload.data.features.map(d => ({
-    ...d,
-    color: [c.r, c.g, c.b, 255 * c.a],
-  }));
-
-  const viewport = {
-    ...fd.viewport,
-    width: slice.width(),
-    height: slice.height(),
-  };
-  // Passing a layer creator function instead of a layer since the
-  // layer needs to be regenerated at each render
-  const layer = () => new ScreenGridLayer({
-    id: `screengrid-layer-${slice.containerId}`,
-    data,
-    pickable: true,
-    cellSizePixels: fd.grid_size,
-    minColor: [c.r, c.g, c.b, 0],
-    maxColor: [c.r, c.g, c.b, 255 * c.a],
-    outline: false,
-    getWeight: d => d.weight || 0,
-  });
-  ReactDOM.render(
-    <DeckGLContainer
-      mapboxApiAccessToken={payload.data.mapboxApiKey}
-      viewport={viewport}
-      layers={[layer]}
-      mapStyle={fd.mapbox_style}
-      setControlValue={setControlValue}
-    />,
-    document.getElementById(slice.containerId),
-  );
-}
-module.exports = deckScreenGridLayer;

superset/assets/visualizations/main.js

@@ -1,4 +1,5 @@
 /* eslint-disable global-require */
+import deckglFactory from './deckgl/factory';
 
 // You ***should*** use these to reference viz_types in code
 export const VIZ_TYPES = {
@@ -44,6 +45,7 @@ export const VIZ_TYPES = {
   deck_hex: 'deck_hex',
   deck_path: 'deck_path',
   deck_geojson: 'deck_geojson',
+  deck_multi: 'deck_multi',
 };
 
 const vizMap = {
@@ -84,11 +86,12 @@ const vizMap = {
   [VIZ_TYPES.event_flow]: require('./EventFlow.jsx'),
   [VIZ_TYPES.paired_ttest]: require('./paired_ttest.jsx'),
   [VIZ_TYPES.partition]: require('./partition.js'),
-  [VIZ_TYPES.deck_scatter]: require('./deckgl/scatter.jsx'),
-  [VIZ_TYPES.deck_screengrid]: require('./deckgl/screengrid.jsx'),
-  [VIZ_TYPES.deck_grid]: require('./deckgl/grid.jsx'),
-  [VIZ_TYPES.deck_hex]: require('./deckgl/hex.jsx'),
-  [VIZ_TYPES.deck_path]: require('./deckgl/path.jsx'),
-  [VIZ_TYPES.deck_geojson]: require('./deckgl/geojson.jsx'),
+  [VIZ_TYPES.deck_scatter]: deckglFactory,
+  [VIZ_TYPES.deck_screengrid]: deckglFactory,
+  [VIZ_TYPES.deck_grid]: deckglFactory,
+  [VIZ_TYPES.deck_hex]: deckglFactory,
+  [VIZ_TYPES.deck_path]: deckglFactory,
+  [VIZ_TYPES.deck_geojson]: deckglFactory,
+  [VIZ_TYPES.deck_multi]: require('./deckgl/multi.jsx'),
 };
 export default vizMap;

superset/views/core.py

@@ -493,7 +493,7 @@ appbuilder.add_view(
 
 class SliceAsync(SliceModelView):  # noqa
     list_columns = [
-        'slice_link', 'viz_type',
+        'id', 'slice_link', 'viz_type', 'slice_name',
         'creator', 'modified', 'icons']
     label_columns = {
         'icons': ' ',

superset/viz.py

@@ -86,6 +86,8 @@ class BaseViz(object):
         """Returns a pandas dataframe based on the query object"""
         if not query_obj:
             query_obj = self.query_obj()
+        if not query_obj:
+            return None
 
         self.error_msg = ''
         self.results = None
@@ -1768,6 +1770,32 @@ class MapboxViz(BaseViz):
         }
 
 
+class DeckGLMultiLayer(BaseViz):
+
+    """Pile on multiple DeckGL layers"""
+
+    viz_type = 'deck_multi'
+    verbose_name = _('Deck.gl - Multiple Layers')
+
+    is_timeseries = False
+    credits = '<a href="https://uber.github.io/deck.gl/">deck.gl</a>'
+
+    def query_obj(self):
+        return None
+
+    def get_data(self, df):
+        fd = self.form_data
+        # Late imports to avoid circular import issues
+        from superset.models.core import Slice
+        from superset import db
+        slice_ids = fd.get('deck_slices')
+        slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all()
+        return {
+            'mapboxApiKey': config.get('MAPBOX_API_KEY'),
+            'slices': [slc.data for slc in slices],
+        }
+
+
 class BaseDeckGLViz(BaseViz):
 
     """Base class for deck.gl visualizations"""