2018-02-25 18:06:11 -05:00
|
|
|
# -*- coding: utf-8 -*-
|
2017-11-14 00:06:51 -05:00
|
|
|
"""This module contains the 'Viz' objects
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
These objects represent the backend of all the visualizations that
|
2016-11-10 02:08:22 -05:00
|
|
|
Superset can render.
|
2016-03-18 02:44:58 -04:00
|
|
|
"""
|
2016-04-07 11:39:08 -04:00
|
|
|
from __future__ import absolute_import
|
|
|
|
from __future__ import division
|
|
|
|
from __future__ import print_function
|
|
|
|
from __future__ import unicode_literals
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-11-07 23:23:40 -05:00
|
|
|
from collections import defaultdict
|
2016-05-10 12:39:33 -04:00
|
|
|
import copy
|
2017-11-07 23:23:40 -05:00
|
|
|
from datetime import datetime, timedelta
|
2016-04-08 02:01:40 -04:00
|
|
|
import hashlib
|
2017-10-04 13:17:33 -04:00
|
|
|
import inspect
|
2017-11-07 23:23:40 -05:00
|
|
|
from itertools import product
|
2016-03-16 23:25:41 -04:00
|
|
|
import logging
|
2018-02-03 23:18:24 -05:00
|
|
|
import math
|
2018-04-11 00:21:24 -04:00
|
|
|
import re
|
2017-02-16 20:28:35 -05:00
|
|
|
import traceback
|
2016-03-18 02:44:58 -04:00
|
|
|
import uuid
|
2016-06-14 00:59:03 -04:00
|
|
|
|
2017-11-07 23:23:40 -05:00
|
|
|
from dateutil import relativedelta as rdelta
|
2018-02-07 17:49:19 -05:00
|
|
|
from flask import escape, request
|
2016-06-27 23:10:40 -04:00
|
|
|
from flask_babel import lazy_gettext as _
|
2017-12-15 14:47:27 -05:00
|
|
|
import geohash
|
2018-02-20 17:41:35 -05:00
|
|
|
from geopy.point import Point
|
2016-03-18 02:44:58 -04:00
|
|
|
from markdown import markdown
|
2017-11-07 23:23:40 -05:00
|
|
|
import numpy as np
|
|
|
|
import pandas as pd
|
2017-12-07 00:50:33 -05:00
|
|
|
from pandas.tseries.frequencies import to_offset
|
2017-12-19 15:38:03 -05:00
|
|
|
import polyline
|
2016-06-11 23:39:25 -04:00
|
|
|
import simplejson as json
|
2018-02-07 17:49:19 -05:00
|
|
|
from six import string_types, text_type
|
|
|
|
from six.moves import cPickle as pkl, reduce
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-11-07 23:23:40 -05:00
|
|
|
from superset import app, cache, get_manifest_file, utils
|
2018-04-04 03:36:23 -04:00
|
|
|
from superset.utils import DTTM_ALIAS, JS_MAX_INTEGER, merge_extra_filters
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2018-03-12 01:07:51 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
config = app.config
|
2017-05-19 13:25:58 -04:00
|
|
|
stats_logger = config.get('STATS_LOGGER')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
class BaseViz(object):
|
|
|
|
|
|
|
|
"""All visualizations derive this base class"""
|
|
|
|
|
|
|
|
viz_type = None
|
2017-11-14 00:06:51 -05:00
|
|
|
verbose_name = 'Base Viz'
|
|
|
|
credits = ''
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2017-09-27 20:51:04 -04:00
|
|
|
default_fillna = 0
|
2018-02-07 17:49:19 -05:00
|
|
|
cache_type = 'df'
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
def __init__(self, datasource, form_data, force=False):
|
2016-04-03 10:37:18 -04:00
|
|
|
if not datasource:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Viz is missing a datasource'))
|
2016-03-18 02:44:58 -04:00
|
|
|
self.datasource = datasource
|
|
|
|
self.request = request
|
2017-11-14 00:06:51 -05:00
|
|
|
self.viz_type = form_data.get('viz_type')
|
2017-02-16 20:28:35 -05:00
|
|
|
self.form_data = form_data
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
self.query = ''
|
2016-03-18 02:44:58 -04:00
|
|
|
self.token = self.form_data.get(
|
|
|
|
'token', 'token_' + uuid.uuid4().hex[:8])
|
2018-03-28 20:41:29 -04:00
|
|
|
metrics = self.form_data.get('metrics') or []
|
|
|
|
self.metrics = []
|
|
|
|
for metric in metrics:
|
|
|
|
if isinstance(metric, dict):
|
|
|
|
self.metrics.append(metric['label'])
|
|
|
|
else:
|
|
|
|
self.metrics.append(metric)
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
self.groupby = self.form_data.get('groupby') or []
|
2017-12-16 19:10:45 -05:00
|
|
|
self.time_shift = timedelta()
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-01-05 13:00:39 -05:00
|
|
|
self.status = None
|
|
|
|
self.error_message = None
|
2018-02-07 17:49:19 -05:00
|
|
|
self.force = force
|
|
|
|
|
|
|
|
# Keeping track of whether some data came from cache
|
|
|
|
# this is useful to trigerr the <CachedLabel /> when
|
|
|
|
# in the cases where visualization have many queries
|
|
|
|
# (FilterBox for instance)
|
|
|
|
self._some_from_cache = False
|
|
|
|
self._any_cache_key = None
|
|
|
|
self._any_cached_dttm = None
|
2018-03-16 12:09:00 -04:00
|
|
|
self._extra_chart_data = None
|
2018-02-07 17:49:19 -05:00
|
|
|
|
2018-04-04 03:36:23 -04:00
|
|
|
@staticmethod
|
|
|
|
def handle_js_int_overflow(data):
|
|
|
|
for d in data.get('records', dict()):
|
|
|
|
for k, v in list(d.items()):
|
|
|
|
if isinstance(v, int):
|
|
|
|
# if an int is too big for Java Script to handle
|
|
|
|
# convert it to a string
|
|
|
|
if abs(v) > JS_MAX_INTEGER:
|
|
|
|
d[k] = str(v)
|
|
|
|
return data
|
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
def run_extra_queries(self):
|
|
|
|
"""Lyfecycle method to use when more than one query is needed
|
|
|
|
|
|
|
|
In rare-ish cases, a visualization may need to execute multiple
|
|
|
|
queries. That is the case for FilterBox or for time comparison
|
|
|
|
in Line chart for instance.
|
|
|
|
|
|
|
|
In those cases, we need to make sure these queries run before the
|
|
|
|
main `get_payload` method gets called, so that the overall caching
|
|
|
|
metadata can be right. The way it works here is that if any of
|
|
|
|
the previous `get_df_payload` calls hit the cache, the main
|
|
|
|
payload's metadata will reflect that.
|
|
|
|
|
|
|
|
The multi-query support may need more work to become a first class
|
|
|
|
use case in the framework, and for the UI to reflect the subtleties
|
|
|
|
(show that only some of the queries were served from cache for
|
|
|
|
instance). In the meantime, since multi-query is rare, we treat
|
|
|
|
it with a bit of a hack. Note that the hack became necessary
|
|
|
|
when moving from caching the visualization's data itself, to caching
|
|
|
|
the underlying query(ies).
|
|
|
|
"""
|
|
|
|
pass
|
2017-01-05 13:00:39 -05:00
|
|
|
|
2018-02-07 11:19:48 -05:00
|
|
|
def get_fillna_for_col(self, col):
|
2017-09-27 20:51:04 -04:00
|
|
|
"""Returns the value for use as filler for a specific Column.type"""
|
2018-02-07 11:19:48 -05:00
|
|
|
if col:
|
|
|
|
if col.is_string:
|
2017-09-27 20:51:04 -04:00
|
|
|
return ' NULL'
|
|
|
|
return self.default_fillna
|
|
|
|
|
|
|
|
def get_fillna_for_columns(self, columns=None):
|
|
|
|
"""Returns a dict or scalar that can be passed to DataFrame.fillna"""
|
|
|
|
if columns is None:
|
|
|
|
return self.default_fillna
|
2018-02-07 11:19:48 -05:00
|
|
|
columns_dict = {col.column_name: col for col in self.datasource.columns}
|
|
|
|
fillna = {
|
|
|
|
c: self.get_fillna_for_col(columns_dict.get(c))
|
|
|
|
for c in columns
|
|
|
|
}
|
2017-09-27 20:51:04 -04:00
|
|
|
return fillna
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def get_df(self, query_obj=None):
|
|
|
|
"""Returns a pandas dataframe based on the query object"""
|
|
|
|
if not query_obj:
|
|
|
|
query_obj = self.query_obj()
|
2017-12-26 13:47:29 -05:00
|
|
|
if not query_obj:
|
|
|
|
return None
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
self.error_msg = ''
|
2016-03-18 02:44:58 -04:00
|
|
|
self.results = None
|
|
|
|
|
2016-06-28 00:33:44 -04:00
|
|
|
timestamp_format = None
|
|
|
|
if self.datasource.type == 'table':
|
|
|
|
dttm_col = self.datasource.get_col(query_obj['granularity'])
|
|
|
|
if dttm_col:
|
|
|
|
timestamp_format = dttm_col.python_date_format
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
# The datasource here can be different backend but the interface is common
|
2017-02-16 20:28:35 -05:00
|
|
|
self.results = self.datasource.query(query_obj)
|
|
|
|
self.query = self.results.query
|
2017-01-05 13:00:39 -05:00
|
|
|
self.status = self.results.status
|
|
|
|
self.error_message = self.results.error_message
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
df = self.results.df
|
2016-06-28 00:33:44 -04:00
|
|
|
# Transform the timestamp we received from database to pandas supported
|
|
|
|
# datetime format. If no python_date_format is specified, the pattern will
|
|
|
|
# be considered as the default ISO date format
|
|
|
|
# If the datetime format is unix, the parse will use the corresponding
|
|
|
|
# parsing logic.
|
2016-03-18 02:44:58 -04:00
|
|
|
if df is None or df.empty:
|
2017-01-05 13:00:39 -05:00
|
|
|
return pd.DataFrame()
|
2016-03-18 02:44:58 -04:00
|
|
|
else:
|
2016-11-15 00:35:10 -05:00
|
|
|
if DTTM_ALIAS in df.columns:
|
2017-11-14 00:06:51 -05:00
|
|
|
if timestamp_format in ('epoch_s', 'epoch_ms'):
|
2018-03-22 11:13:38 -04:00
|
|
|
df[DTTM_ALIAS] = pd.to_datetime(
|
|
|
|
df[DTTM_ALIAS], utc=False, unit=timestamp_format[6:])
|
2016-06-28 00:33:44 -04:00
|
|
|
else:
|
2016-11-15 00:35:10 -05:00
|
|
|
df[DTTM_ALIAS] = pd.to_datetime(
|
|
|
|
df[DTTM_ALIAS], utc=False, format=timestamp_format)
|
2016-03-18 02:44:58 -04:00
|
|
|
if self.datasource.offset:
|
2016-11-15 00:35:10 -05:00
|
|
|
df[DTTM_ALIAS] += timedelta(hours=self.datasource.offset)
|
2017-12-16 19:10:45 -05:00
|
|
|
df[DTTM_ALIAS] += self.time_shift
|
2018-04-03 00:48:14 -04:00
|
|
|
|
|
|
|
self.df_metrics_to_num(df, query_obj.get('metrics') or [])
|
|
|
|
|
2017-01-05 13:00:39 -05:00
|
|
|
df.replace([np.inf, -np.inf], np.nan)
|
2017-09-27 20:51:04 -04:00
|
|
|
fillna = self.get_fillna_for_columns(df.columns)
|
|
|
|
df = df.fillna(fillna)
|
2016-03-18 02:44:58 -04:00
|
|
|
return df
|
|
|
|
|
2018-04-03 00:48:14 -04:00
|
|
|
@staticmethod
|
|
|
|
def df_metrics_to_num(df, metrics):
|
|
|
|
"""Converting metrics to numeric when pandas.read_sql cannot"""
|
|
|
|
for col, dtype in df.dtypes.items():
|
|
|
|
if dtype.type == np.object_ and col in metrics:
|
|
|
|
df[col] = pd.to_numeric(df[col])
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
|
|
|
"""Building a query object"""
|
|
|
|
form_data = self.form_data
|
2017-11-14 00:06:51 -05:00
|
|
|
gb = form_data.get('groupby') or []
|
|
|
|
metrics = form_data.get('metrics') or []
|
|
|
|
columns = form_data.get('columns') or []
|
2017-08-09 21:06:18 -04:00
|
|
|
groupby = []
|
|
|
|
for o in gb + columns:
|
|
|
|
if o not in groupby:
|
|
|
|
groupby.append(o)
|
2017-08-03 18:42:26 -04:00
|
|
|
|
|
|
|
is_timeseries = self.is_timeseries
|
|
|
|
if DTTM_ALIAS in groupby:
|
|
|
|
groupby.remove(DTTM_ALIAS)
|
|
|
|
is_timeseries = True
|
2017-03-14 15:18:33 -04:00
|
|
|
|
2017-10-04 15:43:29 -04:00
|
|
|
# Add extra filters into the query form data
|
|
|
|
merge_extra_filters(form_data)
|
|
|
|
|
2016-09-22 17:30:39 -04:00
|
|
|
granularity = (
|
2017-11-14 00:06:51 -05:00
|
|
|
form_data.get('granularity') or
|
|
|
|
form_data.get('granularity_sqla')
|
2016-09-22 17:30:39 -04:00
|
|
|
)
|
2017-11-14 00:06:51 -05:00
|
|
|
limit = int(form_data.get('limit') or 0)
|
|
|
|
timeseries_limit_metric = form_data.get('timeseries_limit_metric')
|
|
|
|
row_limit = int(form_data.get('row_limit') or config.get('ROW_LIMIT'))
|
2017-03-14 15:18:33 -04:00
|
|
|
|
2017-09-12 12:48:43 -04:00
|
|
|
# default order direction
|
2017-11-14 00:06:51 -05:00
|
|
|
order_desc = form_data.get('order_desc', True)
|
2017-09-12 12:48:43 -04:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
since = form_data.get('since', '')
|
|
|
|
until = form_data.get('until', 'now')
|
2017-12-16 19:10:45 -05:00
|
|
|
time_shift = form_data.get('time_shift', '')
|
2017-03-14 15:18:33 -04:00
|
|
|
|
2017-08-30 14:18:23 -04:00
|
|
|
# Backward compatibility hack
|
2017-10-09 23:59:11 -04:00
|
|
|
if since:
|
|
|
|
since_words = since.split(' ')
|
|
|
|
grains = ['days', 'years', 'hours', 'day', 'year', 'weeks']
|
|
|
|
if (len(since_words) == 2 and since_words[1] in grains):
|
|
|
|
since += ' ago'
|
2017-08-30 14:18:23 -04:00
|
|
|
|
2017-12-16 19:10:45 -05:00
|
|
|
self.time_shift = utils.parse_human_timedelta(time_shift)
|
2017-03-14 15:18:33 -04:00
|
|
|
|
2018-01-09 19:54:18 -05:00
|
|
|
since = utils.parse_human_datetime(since)
|
|
|
|
until = utils.parse_human_datetime(until)
|
|
|
|
from_dttm = None if since is None else (since - self.time_shift)
|
|
|
|
to_dttm = None if until is None else (until - self.time_shift)
|
2017-08-28 12:16:23 -04:00
|
|
|
if from_dttm and to_dttm and from_dttm > to_dttm:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('From date cannot be larger than to date'))
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-09-27 23:40:07 -04:00
|
|
|
self.from_dttm = from_dttm
|
|
|
|
self.to_dttm = to_dttm
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
# extras are used to query elements specific to a datasource type
|
|
|
|
# for instance the extra where clause that applies only to Tables
|
|
|
|
extras = {
|
2017-11-14 00:06:51 -05:00
|
|
|
'where': form_data.get('where', ''),
|
|
|
|
'having': form_data.get('having', ''),
|
2017-09-18 12:44:19 -04:00
|
|
|
'having_druid': form_data.get('having_filters', []),
|
2017-11-14 00:06:51 -05:00
|
|
|
'time_grain_sqla': form_data.get('time_grain_sqla', ''),
|
|
|
|
'druid_time_origin': form_data.get('druid_time_origin', ''),
|
2016-03-18 02:44:58 -04:00
|
|
|
}
|
2017-09-18 12:44:19 -04:00
|
|
|
filters = form_data.get('filters', [])
|
2016-03-18 02:44:58 -04:00
|
|
|
d = {
|
|
|
|
'granularity': granularity,
|
|
|
|
'from_dttm': from_dttm,
|
|
|
|
'to_dttm': to_dttm,
|
2017-08-03 18:42:26 -04:00
|
|
|
'is_timeseries': is_timeseries,
|
2016-03-18 02:44:58 -04:00
|
|
|
'groupby': groupby,
|
|
|
|
'metrics': metrics,
|
|
|
|
'row_limit': row_limit,
|
2017-02-16 20:28:35 -05:00
|
|
|
'filter': filters,
|
2016-03-18 02:44:58 -04:00
|
|
|
'timeseries_limit': limit,
|
|
|
|
'extras': extras,
|
2016-10-17 21:34:26 -04:00
|
|
|
'timeseries_limit_metric': timeseries_limit_metric,
|
2017-11-08 00:32:45 -05:00
|
|
|
'order_desc': order_desc,
|
2018-01-05 16:52:58 -05:00
|
|
|
'prequeries': [],
|
|
|
|
'is_prequery': False,
|
2016-03-18 02:44:58 -04:00
|
|
|
}
|
|
|
|
return d
|
|
|
|
|
2016-03-16 23:25:41 -04:00
|
|
|
@property
|
|
|
|
def cache_timeout(self):
|
2018-04-05 12:58:11 -04:00
|
|
|
if self.form_data.get('cache_timeout'):
|
|
|
|
return int(self.form_data.get('cache_timeout'))
|
2016-03-30 19:28:08 -04:00
|
|
|
if self.datasource.cache_timeout:
|
2016-03-30 19:35:37 -04:00
|
|
|
return self.datasource.cache_timeout
|
2016-04-25 15:41:30 -04:00
|
|
|
if (
|
|
|
|
hasattr(self.datasource, 'database') and
|
|
|
|
self.datasource.database.cache_timeout):
|
2016-03-30 19:35:37 -04:00
|
|
|
return self.datasource.database.cache_timeout
|
2017-11-14 00:06:51 -05:00
|
|
|
return config.get('CACHE_DEFAULT_TIMEOUT')
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
def get_json(self):
|
2017-01-05 13:00:39 -05:00
|
|
|
return json.dumps(
|
2018-02-07 17:49:19 -05:00
|
|
|
self.get_payload(),
|
2017-01-05 13:00:39 -05:00
|
|
|
default=utils.json_int_dttm_ser, ignore_nan=True)
|
|
|
|
|
2018-01-12 15:05:12 -05:00
|
|
|
def cache_key(self, query_obj):
|
|
|
|
"""
|
2018-01-28 12:46:13 -05:00
|
|
|
The cache key is made out of the key/values in `query_obj`
|
|
|
|
|
|
|
|
We remove datetime bounds that are hard values,
|
|
|
|
and replace them with the use-provided inputs to bounds, which
|
|
|
|
may we time-relative (as in "5 days ago" or "now").
|
2018-01-12 15:05:12 -05:00
|
|
|
"""
|
2018-01-28 12:46:13 -05:00
|
|
|
cache_dict = copy.deepcopy(query_obj)
|
|
|
|
|
|
|
|
for k in ['from_dttm', 'to_dttm']:
|
|
|
|
del cache_dict[k]
|
|
|
|
|
2018-03-08 02:29:23 -05:00
|
|
|
for k in ['since', 'until']:
|
2018-01-28 12:46:13 -05:00
|
|
|
cache_dict[k] = self.form_data.get(k)
|
2018-01-12 15:05:12 -05:00
|
|
|
|
2018-03-08 02:29:23 -05:00
|
|
|
cache_dict['datasource'] = self.datasource.uid
|
2018-01-28 12:46:13 -05:00
|
|
|
json_data = self.json_dumps(cache_dict, sort_keys=True)
|
|
|
|
return hashlib.md5(json_data.encode('utf-8')).hexdigest()
|
2017-02-16 20:28:35 -05:00
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
def get_payload(self, query_obj=None):
|
|
|
|
"""Returns a payload of metadata and data"""
|
2018-03-16 12:09:00 -04:00
|
|
|
self.run_extra_queries()
|
2018-02-07 17:49:19 -05:00
|
|
|
payload = self.get_df_payload(query_obj)
|
2018-02-09 18:55:45 -05:00
|
|
|
|
|
|
|
df = payload.get('df')
|
2018-02-20 17:41:35 -05:00
|
|
|
if self.status != utils.QueryStatus.FAILED:
|
2018-03-07 00:19:13 -05:00
|
|
|
if df is not None and df.empty:
|
2018-02-27 18:11:01 -05:00
|
|
|
payload['error'] = 'No data'
|
|
|
|
else:
|
|
|
|
payload['data'] = self.get_data(df)
|
2018-02-20 17:41:35 -05:00
|
|
|
if 'df' in payload:
|
|
|
|
del payload['df']
|
2018-02-07 17:49:19 -05:00
|
|
|
return payload
|
|
|
|
|
|
|
|
def get_df_payload(self, query_obj=None):
|
|
|
|
"""Handles caching around the df payload retrieval"""
|
|
|
|
if not query_obj:
|
|
|
|
query_obj = self.query_obj()
|
2018-01-17 16:54:10 -05:00
|
|
|
cache_key = self.cache_key(query_obj) if query_obj else None
|
2018-02-07 17:49:19 -05:00
|
|
|
logging.info('Cache key: {}'.format(cache_key))
|
|
|
|
is_loaded = False
|
2018-01-12 15:05:12 -05:00
|
|
|
stacktrace = None
|
2018-02-07 17:49:19 -05:00
|
|
|
df = None
|
|
|
|
cached_dttm = datetime.utcnow().isoformat().split('.')[0]
|
|
|
|
if cache_key and cache and not self.force:
|
2018-01-12 15:05:12 -05:00
|
|
|
cache_value = cache.get(cache_key)
|
|
|
|
if cache_value:
|
|
|
|
stats_logger.incr('loaded_from_cache')
|
|
|
|
try:
|
2018-02-07 17:49:19 -05:00
|
|
|
cache_value = pkl.loads(cache_value)
|
|
|
|
df = cache_value['df']
|
2018-03-21 16:13:36 -04:00
|
|
|
self.query = cache_value['query']
|
2018-02-07 17:49:19 -05:00
|
|
|
self._any_cached_dttm = cache_value['dttm']
|
2018-03-21 16:13:36 -04:00
|
|
|
self._any_cache_key = cache_key
|
|
|
|
self.status = utils.QueryStatus.SUCCESS
|
|
|
|
is_loaded = True
|
2018-01-12 15:05:12 -05:00
|
|
|
except Exception as e:
|
2018-02-07 17:49:19 -05:00
|
|
|
logging.exception(e)
|
2018-01-12 15:05:12 -05:00
|
|
|
logging.error('Error reading cache: ' +
|
|
|
|
utils.error_msg_from_exception(e))
|
|
|
|
logging.info('Serving from cache')
|
2016-06-14 00:59:03 -04:00
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
if query_obj and not is_loaded:
|
2017-01-24 17:03:17 -05:00
|
|
|
try:
|
2018-01-12 15:05:12 -05:00
|
|
|
df = self.get_df(query_obj)
|
2018-02-27 18:11:01 -05:00
|
|
|
if self.status != utils.QueryStatus.FAILED:
|
|
|
|
stats_logger.incr('loaded_from_source')
|
|
|
|
is_loaded = True
|
2017-01-24 17:03:17 -05:00
|
|
|
except Exception as e:
|
|
|
|
logging.exception(e)
|
|
|
|
if not self.error_message:
|
2018-02-07 17:49:19 -05:00
|
|
|
self.error_message = escape('{}'.format(e))
|
2017-01-24 17:03:17 -05:00
|
|
|
self.status = utils.QueryStatus.FAILED
|
2017-02-16 20:28:35 -05:00
|
|
|
stacktrace = traceback.format_exc()
|
2018-01-12 15:05:12 -05:00
|
|
|
|
2018-01-18 11:28:26 -05:00
|
|
|
if (
|
2018-02-07 17:49:19 -05:00
|
|
|
is_loaded and
|
2018-01-18 11:28:26 -05:00
|
|
|
cache_key and
|
|
|
|
cache and
|
|
|
|
self.status != utils.QueryStatus.FAILED):
|
2017-02-17 12:50:39 -05:00
|
|
|
try:
|
2018-02-07 17:49:19 -05:00
|
|
|
cache_value = dict(
|
|
|
|
dttm=cached_dttm,
|
|
|
|
df=df if df is not None else None,
|
2018-03-21 16:13:36 -04:00
|
|
|
query=self.query,
|
2018-02-07 17:49:19 -05:00
|
|
|
)
|
|
|
|
cache_value = pkl.dumps(
|
|
|
|
cache_value, protocol=pkl.HIGHEST_PROTOCOL)
|
|
|
|
|
|
|
|
logging.info('Caching {} chars at key {}'.format(
|
|
|
|
len(cache_value), cache_key))
|
|
|
|
|
|
|
|
stats_logger.incr('set_cache_key')
|
2017-02-17 12:50:39 -05:00
|
|
|
cache.set(
|
|
|
|
cache_key,
|
2018-02-07 17:49:19 -05:00
|
|
|
cache_value,
|
2018-01-12 15:05:12 -05:00
|
|
|
timeout=self.cache_timeout)
|
2017-02-17 12:50:39 -05:00
|
|
|
except Exception as e:
|
|
|
|
# cache.set call can fail if the backend is down or if
|
|
|
|
# the key is too large or whatever other reasons
|
2017-11-14 00:06:51 -05:00
|
|
|
logging.warning('Could not cache key {}'.format(cache_key))
|
2017-02-17 12:50:39 -05:00
|
|
|
logging.exception(e)
|
|
|
|
cache.delete(cache_key)
|
2018-01-12 15:05:12 -05:00
|
|
|
|
|
|
|
return {
|
2018-02-07 17:49:19 -05:00
|
|
|
'cache_key': self._any_cache_key,
|
|
|
|
'cached_dttm': self._any_cached_dttm,
|
2018-01-12 15:05:12 -05:00
|
|
|
'cache_timeout': self.cache_timeout,
|
2018-02-07 17:49:19 -05:00
|
|
|
'df': df,
|
2018-01-12 15:05:12 -05:00
|
|
|
'error': self.error_message,
|
|
|
|
'form_data': self.form_data,
|
2018-02-07 17:49:19 -05:00
|
|
|
'is_cached': self._any_cache_key is not None,
|
2018-01-12 15:05:12 -05:00
|
|
|
'query': self.query,
|
|
|
|
'status': self.status,
|
|
|
|
'stacktrace': stacktrace,
|
2018-02-07 17:49:19 -05:00
|
|
|
'rowcount': len(df.index) if df is not None else 0,
|
2018-01-12 15:05:12 -05:00
|
|
|
}
|
2016-04-12 00:22:54 -04:00
|
|
|
|
2018-01-28 12:46:13 -05:00
|
|
|
def json_dumps(self, obj, sort_keys=False):
|
|
|
|
return json.dumps(
|
|
|
|
obj,
|
|
|
|
default=utils.json_int_dttm_ser,
|
|
|
|
ignore_nan=True,
|
|
|
|
sort_keys=sort_keys,
|
|
|
|
)
|
2017-01-12 20:05:34 -05:00
|
|
|
|
2016-04-12 00:22:54 -04:00
|
|
|
@property
|
|
|
|
def data(self):
|
2016-07-13 23:45:05 -04:00
|
|
|
"""This is the data object serialized to the js layer"""
|
2016-04-12 00:22:54 -04:00
|
|
|
content = {
|
|
|
|
'form_data': self.form_data,
|
|
|
|
'token': self.token,
|
|
|
|
'viz_name': self.viz_type,
|
2016-12-16 17:23:48 -05:00
|
|
|
'filter_select_enabled': self.datasource.filter_select_enabled,
|
2016-04-12 00:22:54 -04:00
|
|
|
}
|
|
|
|
return content
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
def get_csv(self):
|
|
|
|
df = self.get_df()
|
2016-04-06 11:22:49 -04:00
|
|
|
include_index = not isinstance(df.index, pd.RangeIndex)
|
2017-09-14 20:40:15 -04:00
|
|
|
return df.to_csv(index=include_index, **config.get('CSV_EXPORT'))
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-16 23:25:41 -04:00
|
|
|
return []
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
@property
|
|
|
|
def json_data(self):
|
2016-06-09 21:05:58 -04:00
|
|
|
return json.dumps(self.data)
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class TableViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A basic html table that is sortable and searchable"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'table'
|
|
|
|
verbose_name = _('Table View')
|
2016-11-10 02:08:22 -05:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
2017-02-27 16:58:05 -05:00
|
|
|
def should_be_timeseries(self):
|
|
|
|
fd = self.form_data
|
|
|
|
# TODO handle datasource-type-specific code in datasource
|
2017-02-28 14:10:42 -05:00
|
|
|
conditions_met = (
|
2017-02-27 16:58:05 -05:00
|
|
|
(fd.get('granularity') and fd.get('granularity') != 'all') or
|
|
|
|
(fd.get('granularity_sqla') and fd.get('time_grain_sqla'))
|
|
|
|
)
|
2017-02-28 14:10:42 -05:00
|
|
|
if fd.get('include_time') and not conditions_met:
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
2017-11-14 00:06:51 -05:00
|
|
|
'Pick a granularity in the Time section or '
|
2017-08-10 23:57:05 -04:00
|
|
|
"uncheck 'Include Time'"))
|
2017-02-28 14:10:42 -05:00
|
|
|
return fd.get('include_time')
|
2017-02-27 16:58:05 -05:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
|
|
|
d = super(TableViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
2017-02-27 16:58:05 -05:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
if fd.get('all_columns') and (fd.get('groupby') or fd.get('metrics')):
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
2017-11-14 00:06:51 -05:00
|
|
|
'Choose either fields to [Group By] and [Metrics] or '
|
|
|
|
'[Columns], not both'))
|
2017-02-27 16:58:05 -05:00
|
|
|
|
2017-09-13 23:18:47 -04:00
|
|
|
sort_by = fd.get('timeseries_limit_metric')
|
2016-03-18 02:44:58 -04:00
|
|
|
if fd.get('all_columns'):
|
|
|
|
d['columns'] = fd.get('all_columns')
|
|
|
|
d['groupby'] = []
|
2016-11-17 10:15:18 -05:00
|
|
|
order_by_cols = fd.get('order_by_cols') or []
|
2016-11-10 18:54:51 -05:00
|
|
|
d['orderby'] = [json.loads(t) for t in order_by_cols]
|
2017-09-13 23:18:47 -04:00
|
|
|
elif sort_by:
|
|
|
|
if sort_by not in d['metrics']:
|
|
|
|
d['metrics'] += [sort_by]
|
2017-11-14 00:06:51 -05:00
|
|
|
d['orderby'] = [(sort_by, not fd.get('order_desc', True))]
|
2017-02-27 16:58:05 -05:00
|
|
|
|
2017-10-16 23:16:20 -04:00
|
|
|
# Add all percent metrics that are not already in the list
|
|
|
|
if 'percent_metrics' in fd:
|
|
|
|
d['metrics'] = d['metrics'] + list(filter(
|
|
|
|
lambda m: m not in d['metrics'],
|
2017-11-08 00:32:45 -05:00
|
|
|
fd['percent_metrics'],
|
2017-10-16 23:16:20 -04:00
|
|
|
))
|
|
|
|
|
2017-02-27 16:58:05 -05:00
|
|
|
d['is_timeseries'] = self.should_be_timeseries()
|
2016-03-18 02:44:58 -04:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2017-10-16 23:16:20 -04:00
|
|
|
fd = self.form_data
|
2018-02-07 17:49:19 -05:00
|
|
|
if (
|
|
|
|
not self.should_be_timeseries() and
|
|
|
|
df is not None and
|
|
|
|
DTTM_ALIAS in df
|
|
|
|
):
|
2016-11-15 00:35:10 -05:00
|
|
|
del df[DTTM_ALIAS]
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-10-16 23:16:20 -04:00
|
|
|
# Sum up and compute percentages for all percent metrics
|
|
|
|
percent_metrics = fd.get('percent_metrics', [])
|
|
|
|
if len(percent_metrics):
|
|
|
|
percent_metrics = list(filter(lambda m: m in df, percent_metrics))
|
|
|
|
metric_sums = {
|
|
|
|
m: reduce(lambda a, b: a + b, df[m])
|
|
|
|
for m in percent_metrics
|
|
|
|
}
|
|
|
|
metric_percents = {
|
|
|
|
m: list(map(lambda a: a / metric_sums[m], df[m]))
|
|
|
|
for m in percent_metrics
|
|
|
|
}
|
|
|
|
for m in percent_metrics:
|
|
|
|
m_name = '%' + m
|
|
|
|
df[m_name] = pd.Series(metric_percents[m], name=m_name)
|
|
|
|
# Remove metrics that are not in the main metrics list
|
|
|
|
for m in filter(
|
|
|
|
lambda m: m not in fd['metrics'] and m in df.columns,
|
2017-11-08 00:32:45 -05:00
|
|
|
percent_metrics,
|
2017-10-16 23:16:20 -04:00
|
|
|
):
|
|
|
|
del df[m]
|
|
|
|
|
2018-04-04 03:36:23 -04:00
|
|
|
data = self.handle_js_int_overflow(
|
|
|
|
dict(
|
|
|
|
records=df.to_dict(orient='records'),
|
|
|
|
columns=list(df.columns),
|
|
|
|
))
|
|
|
|
|
|
|
|
return data
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2018-01-28 12:46:13 -05:00
|
|
|
def json_dumps(self, obj, sort_keys=False):
|
2017-07-26 00:30:40 -04:00
|
|
|
if self.form_data.get('all_columns'):
|
2018-01-28 12:46:13 -05:00
|
|
|
return json.dumps(
|
|
|
|
obj, default=utils.json_iso_dttm_ser, sort_keys=sort_keys)
|
2017-07-26 00:30:40 -04:00
|
|
|
else:
|
|
|
|
return super(TableViz, self).json_dumps(obj)
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-10-04 13:17:33 -04:00
|
|
|
class TimeTableViz(BaseViz):
|
|
|
|
|
|
|
|
"""A data table with rich time-series related columns"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'time_table'
|
|
|
|
verbose_name = _('Time Table View')
|
2017-10-04 13:17:33 -04:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
|
|
|
is_timeseries = True
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(TimeTableViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
|
|
|
|
|
|
|
if not fd.get('metrics'):
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick at least one metric'))
|
2017-10-04 13:17:33 -04:00
|
|
|
|
|
|
|
if fd.get('groupby') and len(fd.get('metrics')) > 1:
|
|
|
|
raise Exception(_(
|
2017-11-14 00:06:51 -05:00
|
|
|
"When using 'Group By' you are limited to use a single metric"))
|
2017-10-04 13:17:33 -04:00
|
|
|
return d
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
values = self.metrics
|
|
|
|
columns = None
|
|
|
|
if fd.get('groupby'):
|
|
|
|
values = self.metrics[0]
|
|
|
|
columns = fd.get('groupby')
|
|
|
|
pt = df.pivot_table(
|
|
|
|
index=DTTM_ALIAS,
|
|
|
|
columns=columns,
|
|
|
|
values=values)
|
|
|
|
pt.index = pt.index.map(str)
|
|
|
|
pt = pt.sort_index()
|
|
|
|
return dict(
|
|
|
|
records=pt.to_dict(orient='index'),
|
|
|
|
columns=list(pt.columns),
|
|
|
|
is_group_by=len(fd.get('groupby')) > 0,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class PivotTableViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A pivot table view, define your rows, columns and metrics"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'pivot_table'
|
|
|
|
verbose_name = _('Pivot Table')
|
2016-11-10 02:08:22 -05:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(PivotTableViz, self).query_obj()
|
|
|
|
groupby = self.form_data.get('groupby')
|
|
|
|
columns = self.form_data.get('columns')
|
|
|
|
metrics = self.form_data.get('metrics')
|
|
|
|
if not columns:
|
|
|
|
columns = []
|
|
|
|
if not groupby:
|
|
|
|
groupby = []
|
|
|
|
if not groupby:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_("Please choose at least one 'Group by' field "))
|
2016-03-18 02:44:58 -04:00
|
|
|
if not metrics:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Please choose at least one metric'))
|
2016-03-18 02:44:58 -04:00
|
|
|
if (
|
|
|
|
any(v in groupby for v in columns) or
|
|
|
|
any(v in columns for v in groupby)):
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_("Group By' and 'Columns' can't overlap"))
|
2016-03-18 02:44:58 -04:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
if (
|
2017-11-14 00:06:51 -05:00
|
|
|
self.form_data.get('granularity') == 'all' and
|
2016-11-15 00:35:10 -05:00
|
|
|
DTTM_ALIAS in df):
|
|
|
|
del df[DTTM_ALIAS]
|
2016-03-18 02:44:58 -04:00
|
|
|
df = df.pivot_table(
|
|
|
|
index=self.form_data.get('groupby'),
|
|
|
|
columns=self.form_data.get('columns'),
|
|
|
|
values=self.form_data.get('metrics'),
|
|
|
|
aggfunc=self.form_data.get('pandas_aggfunc'),
|
2017-07-26 00:11:38 -04:00
|
|
|
margins=self.form_data.get('pivot_margins'),
|
2016-03-18 02:44:58 -04:00
|
|
|
)
|
2017-07-28 17:16:38 -04:00
|
|
|
# Display metrics side by side with each column
|
|
|
|
if self.form_data.get('combine_metric'):
|
|
|
|
df = df.stack(0).unstack()
|
2017-06-20 01:10:54 -04:00
|
|
|
return dict(
|
|
|
|
columns=list(df.columns),
|
|
|
|
html=df.to_html(
|
|
|
|
na_rep='',
|
|
|
|
classes=(
|
2017-11-14 00:06:51 -05:00
|
|
|
'dataframe table table-striped table-bordered '
|
|
|
|
'table-condensed table-hover').split(' ')),
|
2017-06-20 01:10:54 -04:00
|
|
|
)
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
class MarkupViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""Use html or markdown to create a free form widget"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'markup'
|
|
|
|
verbose_name = _('Markup')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
2018-01-17 16:54:10 -05:00
|
|
|
def query_obj(self):
|
|
|
|
return None
|
|
|
|
|
|
|
|
def get_df(self, query_obj=None):
|
2018-03-07 00:19:13 -05:00
|
|
|
return None
|
2017-02-16 20:28:35 -05:00
|
|
|
|
|
|
|
def get_data(self, df):
|
2017-11-14 00:06:51 -05:00
|
|
|
markup_type = self.form_data.get('markup_type')
|
|
|
|
code = self.form_data.get('code', '')
|
|
|
|
if markup_type == 'markdown':
|
2017-01-05 13:00:39 -05:00
|
|
|
code = markdown(code)
|
2017-08-11 00:38:33 -04:00
|
|
|
return dict(html=code, theme_css=get_manifest_file('theme.css'))
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2016-07-13 23:40:52 -04:00
|
|
|
class SeparatorViz(MarkupViz):
|
|
|
|
|
|
|
|
"""Use to create section headers in a dashboard, similar to `Markup`"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'separator'
|
|
|
|
verbose_name = _('Separator')
|
2016-07-13 23:40:52 -04:00
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class WordCloudViz(BaseViz):
|
|
|
|
|
2016-03-16 23:25:41 -04:00
|
|
|
"""Build a colorful word cloud
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2016-03-16 23:25:41 -04:00
|
|
|
Uses the nice library at:
|
2016-03-18 02:44:58 -04:00
|
|
|
https://github.com/jasondavies/d3-cloud
|
|
|
|
"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'word_cloud'
|
|
|
|
verbose_name = _('Word Cloud')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(WordCloudViz, self).query_obj()
|
|
|
|
|
|
|
|
d['metrics'] = [self.form_data.get('metric')]
|
|
|
|
d['groupby'] = [self.form_data.get('series')]
|
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
# Ordering the columns
|
|
|
|
df = df[[self.form_data.get('series'), self.form_data.get('metric')]]
|
|
|
|
# Labeling the columns for uniform json schema
|
|
|
|
df.columns = ['text', 'size']
|
2017-11-14 00:06:51 -05:00
|
|
|
return df.to_dict(orient='records')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2016-04-13 23:27:00 -04:00
|
|
|
class TreemapViz(BaseViz):
|
|
|
|
|
|
|
|
"""Tree map visualisation for hierarchical data."""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'treemap'
|
|
|
|
verbose_name = _('Treemap')
|
2016-04-13 23:27:00 -04:00
|
|
|
credits = '<a href="https://d3js.org">d3.js</a>'
|
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def _nest(self, metric, df):
|
|
|
|
nlevels = df.index.nlevels
|
|
|
|
if nlevels == 1:
|
2017-11-14 00:06:51 -05:00
|
|
|
result = [{'name': n, 'value': v}
|
2016-04-13 23:27:00 -04:00
|
|
|
for n, v in zip(df.index, df[metric])]
|
|
|
|
else:
|
2017-11-14 00:06:51 -05:00
|
|
|
result = [{'name': l, 'children': self._nest(metric, df.loc[l])}
|
2016-04-13 23:27:00 -04:00
|
|
|
for l in df.index.levels[0]]
|
|
|
|
return result
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2017-11-14 00:06:51 -05:00
|
|
|
df = df.set_index(self.form_data.get('groupby'))
|
|
|
|
chart_data = [{'name': metric, 'children': self._nest(metric, df)}
|
2016-04-13 23:27:00 -04:00
|
|
|
for metric in df.columns]
|
|
|
|
return chart_data
|
|
|
|
|
|
|
|
|
2016-05-16 20:59:38 -04:00
|
|
|
class CalHeatmapViz(BaseViz):
|
|
|
|
|
|
|
|
"""Calendar heatmap."""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'cal_heatmap'
|
|
|
|
verbose_name = _('Calendar Heatmap')
|
2016-05-16 20:59:38 -04:00
|
|
|
credits = (
|
|
|
|
'<a href=https://github.com/wa0x6e/cal-heatmap>cal-heatmap</a>')
|
|
|
|
is_timeseries = True
|
2017-02-16 20:28:35 -05:00
|
|
|
|
|
|
|
def get_data(self, df):
|
2016-05-16 20:59:38 -04:00
|
|
|
form_data = self.form_data
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
df.columns = ['timestamp', 'metric']
|
|
|
|
timestamps = {str(obj['timestamp'].value / 10**9):
|
|
|
|
obj.get('metric') for obj in df.to_dict('records')}
|
2016-05-16 20:59:38 -04:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
start = utils.parse_human_datetime(form_data.get('since'))
|
|
|
|
end = utils.parse_human_datetime(form_data.get('until'))
|
|
|
|
domain = form_data.get('domain_granularity')
|
2016-05-16 20:59:38 -04:00
|
|
|
diff_delta = rdelta.relativedelta(end, start)
|
|
|
|
diff_secs = (end - start).total_seconds()
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
if domain == 'year':
|
2016-05-16 20:59:38 -04:00
|
|
|
range_ = diff_delta.years + 1
|
2017-11-14 00:06:51 -05:00
|
|
|
elif domain == 'month':
|
2016-05-16 20:59:38 -04:00
|
|
|
range_ = diff_delta.years * 12 + diff_delta.months + 1
|
2017-11-14 00:06:51 -05:00
|
|
|
elif domain == 'week':
|
2016-05-16 20:59:38 -04:00
|
|
|
range_ = diff_delta.years * 53 + diff_delta.weeks + 1
|
2017-11-14 00:06:51 -05:00
|
|
|
elif domain == 'day':
|
2017-11-08 23:34:23 -05:00
|
|
|
range_ = diff_secs // (24 * 60 * 60) + 1
|
2016-05-16 20:59:38 -04:00
|
|
|
else:
|
2017-11-08 23:34:23 -05:00
|
|
|
range_ = diff_secs // (60 * 60) + 1
|
2016-05-16 20:59:38 -04:00
|
|
|
|
|
|
|
return {
|
2017-11-14 00:06:51 -05:00
|
|
|
'timestamps': timestamps,
|
|
|
|
'start': start,
|
|
|
|
'domain': domain,
|
|
|
|
'subdomain': form_data.get('subdomain_granularity'),
|
|
|
|
'range': range_,
|
2016-05-16 20:59:38 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
qry = super(CalHeatmapViz, self).query_obj()
|
2017-11-14 00:06:51 -05:00
|
|
|
qry['metrics'] = [self.form_data['metric']]
|
2016-05-16 20:59:38 -04:00
|
|
|
return qry
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class NVD3Viz(BaseViz):
|
|
|
|
|
|
|
|
"""Base class for all nvd3 vizs"""
|
|
|
|
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = '<a href="http://nvd3.org/">NVD3.org</a>'
|
2016-03-18 02:44:58 -04:00
|
|
|
viz_type = None
|
2017-11-14 00:06:51 -05:00
|
|
|
verbose_name = 'Base NVD3 Viz'
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
|
2016-04-10 19:15:25 -04:00
|
|
|
class BoxPlotViz(NVD3Viz):
|
|
|
|
|
|
|
|
"""Box plot viz from ND3"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'box_plot'
|
|
|
|
verbose_name = _('Box Plot')
|
2016-04-10 19:15:25 -04:00
|
|
|
sort_series = False
|
2016-09-27 13:11:17 -04:00
|
|
|
is_timeseries = True
|
2016-04-10 19:15:25 -04:00
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def to_series(self, df, classed='', title_suffix=''):
|
2017-11-14 00:06:51 -05:00
|
|
|
label_sep = ' - '
|
2017-02-16 20:28:35 -05:00
|
|
|
chart_data = []
|
2017-11-14 00:06:51 -05:00
|
|
|
for index_value, row in zip(df.index, df.to_dict(orient='records')):
|
2017-02-16 20:28:35 -05:00
|
|
|
if isinstance(index_value, tuple):
|
|
|
|
index_value = label_sep.join(index_value)
|
|
|
|
boxes = defaultdict(dict)
|
|
|
|
for (label, key), value in row.items():
|
2017-11-14 00:06:51 -05:00
|
|
|
if key == 'median':
|
|
|
|
key = 'Q2'
|
2017-02-16 20:28:35 -05:00
|
|
|
boxes[label][key] = value
|
|
|
|
for label, box in boxes.items():
|
2017-11-14 00:06:51 -05:00
|
|
|
if len(self.form_data.get('metrics')) > 1:
|
2017-02-16 20:28:35 -05:00
|
|
|
# need to render data labels with metrics
|
|
|
|
chart_label = label_sep.join([index_value, label])
|
|
|
|
else:
|
|
|
|
chart_label = index_value
|
|
|
|
chart_data.append({
|
2017-11-14 00:06:51 -05:00
|
|
|
'label': chart_label,
|
|
|
|
'values': box,
|
2017-02-16 20:28:35 -05:00
|
|
|
})
|
|
|
|
return chart_data
|
2016-04-10 19:15:25 -04:00
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
|
|
|
form_data = self.form_data
|
2016-04-10 19:15:25 -04:00
|
|
|
df = df.fillna(0)
|
|
|
|
|
|
|
|
# conform to NVD3 names
|
|
|
|
def Q1(series): # need to be named functions - can't use lambdas
|
|
|
|
return np.percentile(series, 25)
|
|
|
|
|
|
|
|
def Q3(series):
|
|
|
|
return np.percentile(series, 75)
|
|
|
|
|
|
|
|
whisker_type = form_data.get('whisker_options')
|
2017-11-14 00:06:51 -05:00
|
|
|
if whisker_type == 'Tukey':
|
2016-04-10 19:15:25 -04:00
|
|
|
|
|
|
|
def whisker_high(series):
|
|
|
|
upper_outer_lim = Q3(series) + 1.5 * (Q3(series) - Q1(series))
|
|
|
|
series = series[series <= upper_outer_lim]
|
|
|
|
return series[np.abs(series - upper_outer_lim).argmin()]
|
|
|
|
|
|
|
|
def whisker_low(series):
|
|
|
|
lower_outer_lim = Q1(series) - 1.5 * (Q3(series) - Q1(series))
|
|
|
|
# find the closest value above the lower outer limit
|
|
|
|
series = series[series >= lower_outer_lim]
|
|
|
|
return series[np.abs(series - lower_outer_lim).argmin()]
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
elif whisker_type == 'Min/max (no outliers)':
|
2016-04-10 19:15:25 -04:00
|
|
|
|
|
|
|
def whisker_high(series):
|
|
|
|
return series.max()
|
|
|
|
|
|
|
|
def whisker_low(series):
|
|
|
|
return series.min()
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
elif ' percentiles' in whisker_type:
|
|
|
|
low, high = whisker_type.replace(' percentiles', '').split('/')
|
2016-04-10 19:15:25 -04:00
|
|
|
|
|
|
|
def whisker_high(series):
|
|
|
|
return np.percentile(series, int(high))
|
|
|
|
|
|
|
|
def whisker_low(series):
|
|
|
|
return np.percentile(series, int(low))
|
|
|
|
|
|
|
|
else:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise ValueError('Unknown whisker type: {}'.format(whisker_type))
|
2016-04-10 19:15:25 -04:00
|
|
|
|
|
|
|
def outliers(series):
|
|
|
|
above = series[series > whisker_high(series)]
|
|
|
|
below = series[series < whisker_low(series)]
|
|
|
|
# pandas sometimes doesn't like getting lists back here
|
|
|
|
return set(above.tolist() + below.tolist())
|
|
|
|
|
|
|
|
aggregate = [Q1, np.median, Q3, whisker_high, whisker_low, outliers]
|
|
|
|
df = df.groupby(form_data.get('groupby')).agg(aggregate)
|
|
|
|
chart_data = self.to_series(df)
|
|
|
|
return chart_data
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class BubbleViz(NVD3Viz):
|
|
|
|
|
|
|
|
"""Based on the NVD3 bubble chart"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'bubble'
|
|
|
|
verbose_name = _('Bubble Chart')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
form_data = self.form_data
|
|
|
|
d = super(BubbleViz, self).query_obj()
|
2017-06-21 15:09:49 -04:00
|
|
|
d['groupby'] = [
|
2017-11-08 00:32:45 -05:00
|
|
|
form_data.get('entity'),
|
2017-06-21 15:09:49 -04:00
|
|
|
]
|
|
|
|
if form_data.get('series'):
|
|
|
|
d['groupby'].append(form_data.get('series'))
|
2016-03-18 02:44:58 -04:00
|
|
|
self.x_metric = form_data.get('x')
|
|
|
|
self.y_metric = form_data.get('y')
|
|
|
|
self.z_metric = form_data.get('size')
|
|
|
|
self.entity = form_data.get('entity')
|
2017-06-21 15:09:49 -04:00
|
|
|
self.series = form_data.get('series') or self.entity
|
2017-05-30 14:04:44 -04:00
|
|
|
d['row_limit'] = form_data.get('limit')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
d['metrics'] = [
|
|
|
|
self.z_metric,
|
|
|
|
self.x_metric,
|
|
|
|
self.y_metric,
|
|
|
|
]
|
2017-06-21 15:09:49 -04:00
|
|
|
if not all(d['metrics'] + [self.entity]):
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick a metric for x, y and size'))
|
2016-03-18 02:44:58 -04:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
df['x'] = df[[self.x_metric]]
|
|
|
|
df['y'] = df[[self.y_metric]]
|
|
|
|
df['size'] = df[[self.z_metric]]
|
|
|
|
df['shape'] = 'circle'
|
|
|
|
df['group'] = df[[self.series]]
|
|
|
|
|
|
|
|
series = defaultdict(list)
|
|
|
|
for row in df.to_dict(orient='records'):
|
|
|
|
series[row['group']].append(row)
|
|
|
|
chart_data = []
|
|
|
|
for k, v in series.items():
|
|
|
|
chart_data.append({
|
|
|
|
'key': k,
|
2016-03-16 23:25:41 -04:00
|
|
|
'values': v})
|
|
|
|
return chart_data
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2016-12-12 11:58:13 -05:00
|
|
|
class BulletViz(NVD3Viz):
|
|
|
|
|
|
|
|
"""Based on the NVD3 bullet chart"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'bullet'
|
|
|
|
verbose_name = _('Bullet Chart')
|
2016-12-12 11:58:13 -05:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
form_data = self.form_data
|
|
|
|
d = super(BulletViz, self).query_obj()
|
|
|
|
self.metric = form_data.get('metric')
|
|
|
|
|
|
|
|
def as_strings(field):
|
|
|
|
value = form_data.get(field)
|
|
|
|
return value.split(',') if value else []
|
|
|
|
|
|
|
|
def as_floats(field):
|
|
|
|
return [float(x) for x in as_strings(field)]
|
|
|
|
|
|
|
|
self.ranges = as_floats('ranges')
|
|
|
|
self.range_labels = as_strings('range_labels')
|
|
|
|
self.markers = as_floats('markers')
|
|
|
|
self.marker_labels = as_strings('marker_labels')
|
|
|
|
self.marker_lines = as_floats('marker_lines')
|
|
|
|
self.marker_line_labels = as_strings('marker_line_labels')
|
|
|
|
|
|
|
|
d['metrics'] = [
|
|
|
|
self.metric,
|
|
|
|
]
|
|
|
|
if not self.metric:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick a metric to display'))
|
2016-12-12 11:58:13 -05:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-12-12 11:58:13 -05:00
|
|
|
df = df.fillna(0)
|
|
|
|
df['metric'] = df[[self.metric]]
|
|
|
|
values = df['metric'].values
|
|
|
|
return {
|
|
|
|
'measures': values.tolist(),
|
|
|
|
'ranges': self.ranges or [0, values.max() * 1.1],
|
|
|
|
'rangeLabels': self.range_labels or None,
|
|
|
|
'markers': self.markers or None,
|
|
|
|
'markerLabels': self.marker_labels or None,
|
|
|
|
'markerLines': self.marker_lines or None,
|
|
|
|
'markerLineLabels': self.marker_line_labels or None,
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class BigNumberViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""Put emphasis on a single metric with this big number viz"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'big_number'
|
|
|
|
verbose_name = _('Big Number with Trendline')
|
2016-11-10 02:08:22 -05:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = True
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(BigNumberViz, self).query_obj()
|
|
|
|
metric = self.form_data.get('metric')
|
|
|
|
if not metric:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick a metric!'))
|
2016-03-18 02:44:58 -04:00
|
|
|
d['metrics'] = [self.form_data.get('metric')]
|
|
|
|
self.form_data['metric'] = metric
|
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
form_data = self.form_data
|
2016-06-09 21:05:58 -04:00
|
|
|
df.sort_values(by=df.columns[0], inplace=True)
|
2017-11-14 00:06:51 -05:00
|
|
|
compare_lag = form_data.get('compare_lag')
|
2016-03-16 23:25:41 -04:00
|
|
|
return {
|
2016-03-18 02:44:58 -04:00
|
|
|
'data': df.values.tolist(),
|
|
|
|
'compare_lag': compare_lag,
|
|
|
|
'compare_suffix': form_data.get('compare_suffix', ''),
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2016-04-07 17:22:12 -04:00
|
|
|
class BigNumberTotalViz(BaseViz):
|
|
|
|
|
|
|
|
"""Put emphasis on a single metric with this big number viz"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'big_number_total'
|
|
|
|
verbose_name = _('Big Number')
|
2016-11-10 02:08:22 -05:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
2016-04-07 17:22:12 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(BigNumberTotalViz, self).query_obj()
|
|
|
|
metric = self.form_data.get('metric')
|
|
|
|
if not metric:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick a metric!'))
|
2016-04-07 17:22:12 -04:00
|
|
|
d['metrics'] = [self.form_data.get('metric')]
|
|
|
|
self.form_data['metric'] = metric
|
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-04-07 17:22:12 -04:00
|
|
|
form_data = self.form_data
|
2016-06-09 21:05:58 -04:00
|
|
|
df.sort_values(by=df.columns[0], inplace=True)
|
2016-04-07 17:22:12 -04:00
|
|
|
return {
|
|
|
|
'data': df.values.tolist(),
|
|
|
|
'subheader': form_data.get('subheader', ''),
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class NVD3TimeSeriesViz(NVD3Viz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A rich line chart component with tons of options"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'line'
|
|
|
|
verbose_name = _('Time Series - Line Chart')
|
2016-03-18 02:44:58 -04:00
|
|
|
sort_series = False
|
|
|
|
is_timeseries = True
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def to_series(self, df, classed='', title_suffix=''):
|
|
|
|
cols = []
|
|
|
|
for col in df.columns:
|
|
|
|
if col == '':
|
|
|
|
cols.append('N/A')
|
|
|
|
elif col is None:
|
|
|
|
cols.append('NULL')
|
|
|
|
else:
|
|
|
|
cols.append(col)
|
|
|
|
df.columns = cols
|
|
|
|
series = df.to_dict('series')
|
|
|
|
|
|
|
|
chart_data = []
|
|
|
|
for name in df.T.index.tolist():
|
|
|
|
ys = series[name]
|
2017-11-14 00:06:51 -05:00
|
|
|
if df[name].dtype.kind not in 'biufc':
|
2017-02-16 20:28:35 -05:00
|
|
|
continue
|
2018-04-04 03:36:23 -04:00
|
|
|
if isinstance(name, list):
|
|
|
|
series_title = [str(title) for title in name]
|
|
|
|
elif isinstance(name, tuple):
|
|
|
|
series_title = tuple(str(title) for title in name)
|
|
|
|
else:
|
|
|
|
series_title = str(name)
|
2017-10-02 13:42:16 -04:00
|
|
|
if (
|
|
|
|
isinstance(series_title, (list, tuple)) and
|
|
|
|
len(series_title) > 1 and
|
|
|
|
len(self.metrics) == 1):
|
|
|
|
# Removing metric from series name if only one metric
|
|
|
|
series_title = series_title[1:]
|
2018-04-07 18:19:58 -04:00
|
|
|
if title_suffix:
|
|
|
|
if isinstance(series_title, string_types):
|
|
|
|
series_title = (series_title, title_suffix)
|
|
|
|
elif isinstance(series_title, (list, tuple)):
|
|
|
|
series_title = series_title + (title_suffix,)
|
2017-02-16 20:28:35 -05:00
|
|
|
|
2017-12-07 00:50:33 -05:00
|
|
|
values = []
|
|
|
|
for ds in df.index:
|
|
|
|
if ds in ys:
|
|
|
|
d = {
|
|
|
|
'x': ds,
|
|
|
|
'y': ys[ds],
|
|
|
|
}
|
|
|
|
else:
|
|
|
|
d = {}
|
|
|
|
values.append(d)
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
d = {
|
2017-11-14 00:06:51 -05:00
|
|
|
'key': series_title,
|
2017-12-07 00:50:33 -05:00
|
|
|
'values': values,
|
2017-02-16 20:28:35 -05:00
|
|
|
}
|
2017-12-07 00:50:33 -05:00
|
|
|
if classed:
|
|
|
|
d['classed'] = classed
|
2017-02-16 20:28:35 -05:00
|
|
|
chart_data.append(d)
|
|
|
|
return chart_data
|
|
|
|
|
2017-10-13 00:54:59 -04:00
|
|
|
def process_data(self, df, aggregate=False):
|
2017-02-16 20:28:35 -05:00
|
|
|
fd = self.form_data
|
2016-03-18 02:44:58 -04:00
|
|
|
df = df.fillna(0)
|
2017-11-14 00:06:51 -05:00
|
|
|
if fd.get('granularity') == 'all':
|
|
|
|
raise Exception(_('Pick a time granularity for your time series'))
|
2017-10-13 00:54:59 -04:00
|
|
|
if not aggregate:
|
|
|
|
df = df.pivot_table(
|
|
|
|
index=DTTM_ALIAS,
|
|
|
|
columns=fd.get('groupby'),
|
2018-03-28 20:41:29 -04:00
|
|
|
values=utils.get_metric_names(fd.get('metrics')))
|
2017-10-13 00:54:59 -04:00
|
|
|
else:
|
|
|
|
df = df.pivot_table(
|
|
|
|
index=DTTM_ALIAS,
|
|
|
|
columns=fd.get('groupby'),
|
2018-03-28 20:41:29 -04:00
|
|
|
values=utils.get_metric_names(fd.get('metrics')),
|
2017-10-13 00:54:59 -04:00
|
|
|
fill_value=0,
|
|
|
|
aggfunc=sum)
|
2016-03-18 02:44:58 -04:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
fm = fd.get('resample_fillmethod')
|
2016-03-18 02:44:58 -04:00
|
|
|
if not fm:
|
|
|
|
fm = None
|
2017-11-14 00:06:51 -05:00
|
|
|
how = fd.get('resample_how')
|
|
|
|
rule = fd.get('resample_rule')
|
2016-03-18 02:44:58 -04:00
|
|
|
if how and rule:
|
|
|
|
df = df.resample(rule, how=how, fill_method=fm)
|
|
|
|
if not fm:
|
|
|
|
df = df.fillna(0)
|
|
|
|
|
|
|
|
if self.sort_series:
|
|
|
|
dfs = df.sum()
|
2016-08-02 02:02:16 -04:00
|
|
|
dfs.sort_values(ascending=False, inplace=True)
|
2016-03-18 02:44:58 -04:00
|
|
|
df = df[dfs.index]
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
if fd.get('contribution'):
|
2016-03-18 02:44:58 -04:00
|
|
|
dft = df.T
|
|
|
|
df = (dft / dft.sum()).T
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
rolling_type = fd.get('rolling_type')
|
|
|
|
rolling_periods = int(fd.get('rolling_periods') or 0)
|
|
|
|
min_periods = int(fd.get('min_periods') or 0)
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
if rolling_type in ('mean', 'std', 'sum') and rolling_periods:
|
2017-08-30 16:32:07 -04:00
|
|
|
kwargs = dict(
|
|
|
|
arg=df,
|
|
|
|
window=rolling_periods,
|
|
|
|
min_periods=min_periods)
|
2016-03-18 02:44:58 -04:00
|
|
|
if rolling_type == 'mean':
|
2017-08-30 16:32:07 -04:00
|
|
|
df = pd.rolling_mean(**kwargs)
|
2016-03-18 02:44:58 -04:00
|
|
|
elif rolling_type == 'std':
|
2017-08-30 16:32:07 -04:00
|
|
|
df = pd.rolling_std(**kwargs)
|
2016-03-18 02:44:58 -04:00
|
|
|
elif rolling_type == 'sum':
|
2017-08-30 16:32:07 -04:00
|
|
|
df = pd.rolling_sum(**kwargs)
|
2016-03-18 02:44:58 -04:00
|
|
|
elif rolling_type == 'cumsum':
|
|
|
|
df = df.cumsum()
|
2017-08-30 16:32:07 -04:00
|
|
|
if min_periods:
|
|
|
|
df = df[min_periods:]
|
2017-01-11 00:09:02 -05:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
num_period_compare = fd.get('num_period_compare')
|
2017-01-11 00:09:02 -05:00
|
|
|
if num_period_compare:
|
|
|
|
num_period_compare = int(num_period_compare)
|
2017-02-16 20:28:35 -05:00
|
|
|
prt = fd.get('period_ratio_type')
|
2017-01-11 00:09:02 -05:00
|
|
|
if prt and prt == 'growth':
|
|
|
|
df = (df / df.shift(num_period_compare)) - 1
|
|
|
|
elif prt and prt == 'value':
|
|
|
|
df = df - df.shift(num_period_compare)
|
|
|
|
else:
|
|
|
|
df = df / df.shift(num_period_compare)
|
|
|
|
|
|
|
|
df = df[num_period_compare:]
|
2017-08-25 14:08:15 -04:00
|
|
|
return df
|
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
def run_extra_queries(self):
|
2017-08-25 14:08:15 -04:00
|
|
|
fd = self.form_data
|
2017-02-16 20:28:35 -05:00
|
|
|
time_compare = fd.get('time_compare')
|
2016-03-18 02:44:58 -04:00
|
|
|
if time_compare:
|
|
|
|
query_object = self.query_obj()
|
|
|
|
delta = utils.parse_human_timedelta(time_compare)
|
|
|
|
query_object['inner_from_dttm'] = query_object['from_dttm']
|
|
|
|
query_object['inner_to_dttm'] = query_object['to_dttm']
|
2018-02-07 17:33:04 -05:00
|
|
|
|
|
|
|
if not query_object['from_dttm'] or not query_object['to_dttm']:
|
|
|
|
raise Exception(_(
|
|
|
|
'`Since` and `Until` time bounds should be specified '
|
|
|
|
'when using the `Time Shift` feature.'))
|
2016-03-18 02:44:58 -04:00
|
|
|
query_object['from_dttm'] -= delta
|
|
|
|
query_object['to_dttm'] -= delta
|
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
df2 = self.get_df_payload(query_object).get('df')
|
2018-02-12 14:48:14 -05:00
|
|
|
if df2 is not None:
|
|
|
|
df2[DTTM_ALIAS] += delta
|
|
|
|
df2 = self.process_data(df2)
|
2018-03-16 12:09:00 -04:00
|
|
|
self._extra_chart_data = self.to_series(
|
2018-02-12 14:48:14 -05:00
|
|
|
df2, classed='superset', title_suffix='---')
|
2018-02-07 17:49:19 -05:00
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
df = self.process_data(df)
|
|
|
|
chart_data = self.to_series(df)
|
|
|
|
|
2018-03-16 12:09:00 -04:00
|
|
|
if self._extra_chart_data:
|
|
|
|
chart_data += self._extra_chart_data
|
2018-03-26 13:55:43 -04:00
|
|
|
chart_data = sorted(chart_data, key=lambda x: tuple(x['key']))
|
2018-02-07 17:49:19 -05:00
|
|
|
|
2016-03-16 23:25:41 -04:00
|
|
|
return chart_data
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2017-01-06 13:30:27 -05:00
|
|
|
class NVD3DualLineViz(NVD3Viz):
|
|
|
|
|
|
|
|
"""A rich line chart with dual axis"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'dual_line'
|
|
|
|
verbose_name = _('Time Series - Dual Axis Line Chart')
|
2017-01-06 13:30:27 -05:00
|
|
|
sort_series = False
|
|
|
|
is_timeseries = True
|
|
|
|
|
2017-01-11 13:07:30 -05:00
|
|
|
def query_obj(self):
|
|
|
|
d = super(NVD3DualLineViz, self).query_obj()
|
2017-02-16 20:28:35 -05:00
|
|
|
m1 = self.form_data.get('metric')
|
|
|
|
m2 = self.form_data.get('metric_2')
|
|
|
|
d['metrics'] = [m1, m2]
|
|
|
|
if not m1:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick a metric for left axis!'))
|
2017-02-16 20:28:35 -05:00
|
|
|
if not m2:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick a metric for right axis!'))
|
2017-02-16 20:28:35 -05:00
|
|
|
if m1 == m2:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Please choose different metrics'
|
|
|
|
' on left and right axis'))
|
2017-01-11 13:07:30 -05:00
|
|
|
return d
|
|
|
|
|
2017-01-06 13:30:27 -05:00
|
|
|
def to_series(self, df, classed=''):
|
|
|
|
cols = []
|
|
|
|
for col in df.columns:
|
|
|
|
if col == '':
|
|
|
|
cols.append('N/A')
|
|
|
|
elif col is None:
|
|
|
|
cols.append('NULL')
|
|
|
|
else:
|
|
|
|
cols.append(col)
|
|
|
|
df.columns = cols
|
|
|
|
series = df.to_dict('series')
|
|
|
|
chart_data = []
|
2017-02-10 15:54:03 -05:00
|
|
|
metrics = [
|
|
|
|
self.form_data.get('metric'),
|
2017-11-08 00:32:45 -05:00
|
|
|
self.form_data.get('metric_2'),
|
2017-02-10 15:54:03 -05:00
|
|
|
]
|
|
|
|
for i, m in enumerate(metrics):
|
|
|
|
ys = series[m]
|
2017-11-14 00:06:51 -05:00
|
|
|
if df[m].dtype.kind not in 'biufc':
|
2017-01-06 13:30:27 -05:00
|
|
|
continue
|
2017-02-10 15:54:03 -05:00
|
|
|
series_title = m
|
2017-01-06 13:30:27 -05:00
|
|
|
d = {
|
2017-11-14 00:06:51 -05:00
|
|
|
'key': series_title,
|
|
|
|
'classed': classed,
|
|
|
|
'values': [
|
2017-01-06 13:30:27 -05:00
|
|
|
{'x': ds, 'y': ys[ds] if ds in ys else None}
|
2017-02-17 01:06:23 -05:00
|
|
|
for ds in df.index
|
2017-01-06 13:30:27 -05:00
|
|
|
],
|
2017-11-14 00:06:51 -05:00
|
|
|
'yAxis': i + 1,
|
|
|
|
'type': 'line',
|
2017-01-06 13:30:27 -05:00
|
|
|
}
|
|
|
|
chart_data.append(d)
|
|
|
|
return chart_data
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
df = df.fillna(0)
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
if self.form_data.get('granularity') == 'all':
|
|
|
|
raise Exception(_('Pick a time granularity for your time series'))
|
2017-02-16 20:28:35 -05:00
|
|
|
|
|
|
|
metric = fd.get('metric')
|
|
|
|
metric_2 = fd.get('metric_2')
|
|
|
|
df = df.pivot_table(
|
|
|
|
index=DTTM_ALIAS,
|
|
|
|
values=[metric, metric_2])
|
2017-01-06 13:30:27 -05:00
|
|
|
|
|
|
|
chart_data = self.to_series(df)
|
|
|
|
return chart_data
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class NVD3TimeSeriesBarViz(NVD3TimeSeriesViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A bar chart where the x axis is time"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'bar'
|
2016-03-18 02:44:58 -04:00
|
|
|
sort_series = True
|
2017-11-14 00:06:51 -05:00
|
|
|
verbose_name = _('Time Series - Bar Chart')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2017-12-07 00:50:33 -05:00
|
|
|
class NVD3TimePivotViz(NVD3TimeSeriesViz):
|
|
|
|
|
|
|
|
"""Time Series - Periodicity Pivot"""
|
|
|
|
|
|
|
|
viz_type = 'time_pivot'
|
|
|
|
sort_series = True
|
|
|
|
verbose_name = _('Time Series - Period Pivot')
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(NVD3TimePivotViz, self).query_obj()
|
|
|
|
d['metrics'] = [self.form_data.get('metric')]
|
|
|
|
return d
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
df = self.process_data(df)
|
|
|
|
freq = to_offset(fd.get('freq'))
|
|
|
|
freq.normalize = True
|
|
|
|
df[DTTM_ALIAS] = df.index.map(freq.rollback)
|
|
|
|
df['ranked'] = df[DTTM_ALIAS].rank(method='dense', ascending=False) - 1
|
|
|
|
df.ranked = df.ranked.map(int)
|
|
|
|
df['series'] = '-' + df.ranked.map(str)
|
|
|
|
df['series'] = df['series'].str.replace('-0', 'current')
|
|
|
|
rank_lookup = {
|
|
|
|
row['series']: row['ranked']
|
|
|
|
for row in df.to_dict(orient='records')
|
|
|
|
}
|
|
|
|
max_ts = df[DTTM_ALIAS].max()
|
|
|
|
max_rank = df['ranked'].max()
|
|
|
|
df[DTTM_ALIAS] = df.index + (max_ts - df[DTTM_ALIAS])
|
|
|
|
df = df.pivot_table(
|
|
|
|
index=DTTM_ALIAS,
|
|
|
|
columns='series',
|
|
|
|
values=fd.get('metric'))
|
|
|
|
chart_data = self.to_series(df)
|
|
|
|
for serie in chart_data:
|
|
|
|
serie['rank'] = rank_lookup[serie['key']]
|
|
|
|
serie['perc'] = 1 - (serie['rank'] / (max_rank + 1))
|
|
|
|
return chart_data
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class NVD3CompareTimeSeriesViz(NVD3TimeSeriesViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A line chart component where you can compare the % change over time"""
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
viz_type = 'compare'
|
2017-11-14 00:06:51 -05:00
|
|
|
verbose_name = _('Time Series - Percent Change')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
class NVD3TimeSeriesStackedViz(NVD3TimeSeriesViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A rich stack area chart"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'area'
|
|
|
|
verbose_name = _('Time Series - Stacked')
|
2016-03-18 02:44:58 -04:00
|
|
|
sort_series = True
|
|
|
|
|
|
|
|
|
|
|
|
class DistributionPieViz(NVD3Viz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""Annoy visualization snobs with this controversial pie chart"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'pie'
|
|
|
|
verbose_name = _('Distribution - NVD3 - Pie Chart')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
df = df.pivot_table(
|
|
|
|
index=self.groupby,
|
|
|
|
values=[self.metrics[0]])
|
2016-06-09 21:05:58 -04:00
|
|
|
df.sort_values(by=self.metrics[0], ascending=False, inplace=True)
|
2017-03-21 00:10:06 -04:00
|
|
|
df = df.reset_index()
|
2016-03-18 02:44:58 -04:00
|
|
|
df.columns = ['x', 'y']
|
2017-11-14 00:06:51 -05:00
|
|
|
return df.to_dict(orient='records')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2016-08-10 18:04:19 -04:00
|
|
|
class HistogramViz(BaseViz):
|
|
|
|
|
|
|
|
"""Histogram"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'histogram'
|
|
|
|
verbose_name = _('Histogram')
|
2016-08-10 18:04:19 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
"""Returns the query object for this visualization"""
|
|
|
|
d = super(HistogramViz, self).query_obj()
|
2017-01-31 20:50:53 -05:00
|
|
|
d['row_limit'] = self.form_data.get(
|
|
|
|
'row_limit', int(config.get('VIZ_ROW_LIMIT')))
|
2018-04-11 00:21:24 -04:00
|
|
|
numeric_columns = self.form_data.get('all_columns_x')
|
|
|
|
if numeric_columns is None:
|
|
|
|
raise Exception(_('Must have at least one numeric column specified'))
|
|
|
|
self.columns = numeric_columns
|
|
|
|
d['columns'] = numeric_columns + self.groupby
|
|
|
|
# override groupby entry to avoid aggregation
|
|
|
|
d['groupby'] = []
|
2016-08-10 18:04:19 -04:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-08-10 18:04:19 -04:00
|
|
|
"""Returns the chart data"""
|
2018-04-11 00:21:24 -04:00
|
|
|
chart_data = []
|
|
|
|
if len(self.groupby) > 0:
|
|
|
|
groups = df.groupby(self.groupby)
|
|
|
|
else:
|
|
|
|
groups = [((), df)]
|
|
|
|
for keys, data in groups:
|
|
|
|
if isinstance(keys, str):
|
|
|
|
keys = (keys,)
|
|
|
|
# removing undesirable characters
|
|
|
|
keys = [re.sub(r'\W+', r'_', k) for k in keys]
|
|
|
|
chart_data.extend([{
|
|
|
|
'key': '__'.join([c] + keys),
|
|
|
|
'values': data[c].tolist()}
|
|
|
|
for c in self.columns])
|
2016-08-10 18:04:19 -04:00
|
|
|
return chart_data
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class DistributionBarViz(DistributionPieViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A good old bar chart"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'dist_bar'
|
|
|
|
verbose_name = _('Distribution - Bar Chart')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
2017-04-04 00:53:06 -04:00
|
|
|
d = super(DistributionBarViz, self).query_obj() # noqa
|
2016-03-18 02:44:58 -04:00
|
|
|
fd = self.form_data
|
2017-08-03 18:42:26 -04:00
|
|
|
if (
|
2017-11-10 15:06:22 -05:00
|
|
|
len(d['groupby']) <
|
|
|
|
len(fd.get('groupby') or []) + len(fd.get('columns') or [])
|
|
|
|
):
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(
|
|
|
|
_("Can't have overlap between Series and Breakdowns"))
|
2017-08-08 00:47:42 -04:00
|
|
|
if not fd.get('metrics'):
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick at least one metric'))
|
2017-08-08 00:47:42 -04:00
|
|
|
if not fd.get('groupby'):
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick at least one field for [Series]'))
|
2016-03-18 02:44:58 -04:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
fd = self.form_data
|
|
|
|
|
|
|
|
row = df.groupby(self.groupby).sum()[self.metrics[0]].copy()
|
2016-08-02 02:02:16 -04:00
|
|
|
row.sort_values(ascending=False, inplace=True)
|
2016-03-18 02:44:58 -04:00
|
|
|
columns = fd.get('columns') or []
|
|
|
|
pt = df.pivot_table(
|
|
|
|
index=self.groupby,
|
|
|
|
columns=columns,
|
|
|
|
values=self.metrics)
|
2017-11-14 00:06:51 -05:00
|
|
|
if fd.get('contribution'):
|
2016-06-20 12:16:51 -04:00
|
|
|
pt = pt.fillna(0)
|
|
|
|
pt = pt.T
|
|
|
|
pt = (pt / pt.sum()).T
|
2016-03-18 02:44:58 -04:00
|
|
|
pt = pt.reindex(row.index)
|
|
|
|
chart_data = []
|
2018-04-03 00:48:14 -04:00
|
|
|
for name, ys in pt.items():
|
2017-11-14 00:06:51 -05:00
|
|
|
if pt[name].dtype.kind not in 'biufc' or name in self.groupby:
|
2016-03-18 02:44:58 -04:00
|
|
|
continue
|
|
|
|
if isinstance(name, string_types):
|
|
|
|
series_title = name
|
|
|
|
elif len(self.metrics) > 1:
|
2017-11-14 00:06:51 -05:00
|
|
|
series_title = ', '.join(name)
|
2016-03-18 02:44:58 -04:00
|
|
|
else:
|
2017-11-09 23:23:59 -05:00
|
|
|
l = [str(s) for s in name[1:]] # noqa: E741
|
2017-11-14 00:06:51 -05:00
|
|
|
series_title = ', '.join(l)
|
2017-02-27 02:44:08 -05:00
|
|
|
values = []
|
2018-04-03 00:48:14 -04:00
|
|
|
for i, v in ys.items():
|
2017-03-16 02:50:12 -04:00
|
|
|
x = i
|
|
|
|
if isinstance(x, (tuple, list)):
|
2017-12-09 17:23:24 -05:00
|
|
|
x = ', '.join([text_type(s) for s in x])
|
2017-03-16 02:50:12 -04:00
|
|
|
else:
|
2017-12-09 17:23:24 -05:00
|
|
|
x = text_type(x)
|
2017-03-16 02:50:12 -04:00
|
|
|
values.append({
|
|
|
|
'x': x,
|
|
|
|
'y': v,
|
|
|
|
})
|
2016-03-18 02:44:58 -04:00
|
|
|
d = {
|
2017-11-14 00:06:51 -05:00
|
|
|
'key': series_title,
|
|
|
|
'values': values,
|
2016-03-18 02:44:58 -04:00
|
|
|
}
|
|
|
|
chart_data.append(d)
|
2016-03-16 23:25:41 -04:00
|
|
|
return chart_data
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
class SunburstViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A multi level sunburst chart"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'sunburst'
|
|
|
|
verbose_name = _('Sunburst')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = (
|
|
|
|
'Kerry Rodden '
|
|
|
|
'@<a href="https://bl.ocks.org/kerryrodden/7090426">bl.ocks.org</a>')
|
2017-02-16 20:28:35 -05:00
|
|
|
|
|
|
|
def get_data(self, df):
|
2018-02-09 17:27:22 -05:00
|
|
|
fd = self.form_data
|
|
|
|
cols = fd.get('groupby')
|
|
|
|
metric = fd.get('metric')
|
|
|
|
secondary_metric = fd.get('secondary_metric')
|
|
|
|
if metric == secondary_metric or secondary_metric is None:
|
|
|
|
df.columns = cols + ['m1']
|
|
|
|
df['m2'] = df['m1']
|
|
|
|
return json.loads(df.to_json(orient='values'))
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
qry = super(SunburstViz, self).query_obj()
|
2018-02-09 17:27:22 -05:00
|
|
|
fd = self.form_data
|
|
|
|
qry['metrics'] = [fd['metric']]
|
|
|
|
secondary_metric = fd.get('secondary_metric')
|
|
|
|
if secondary_metric and secondary_metric != fd['metric']:
|
|
|
|
qry['metrics'].append(secondary_metric)
|
2016-03-18 02:44:58 -04:00
|
|
|
return qry
|
|
|
|
|
|
|
|
|
|
|
|
class SankeyViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A Sankey diagram that requires a parent-child dataset"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'sankey'
|
|
|
|
verbose_name = _('Sankey')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = '<a href="https://www.npmjs.com/package/d3-sankey">d3-sankey on npm</a>'
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
qry = super(SankeyViz, self).query_obj()
|
|
|
|
if len(qry['groupby']) != 2:
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick exactly 2 columns as [Source / Target]'))
|
2016-03-18 02:44:58 -04:00
|
|
|
qry['metrics'] = [
|
|
|
|
self.form_data['metric']]
|
|
|
|
return qry
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
df.columns = ['source', 'target', 'value']
|
2016-04-09 03:17:31 -04:00
|
|
|
recs = df.to_dict(orient='records')
|
|
|
|
|
|
|
|
hierarchy = defaultdict(set)
|
|
|
|
for row in recs:
|
|
|
|
hierarchy[row['source']].add(row['target'])
|
|
|
|
|
|
|
|
def find_cycle(g):
|
|
|
|
"""Whether there's a cycle in a directed graph"""
|
|
|
|
path = set()
|
2016-04-11 01:49:08 -04:00
|
|
|
|
2016-04-09 03:17:31 -04:00
|
|
|
def visit(vertex):
|
|
|
|
path.add(vertex)
|
|
|
|
for neighbour in g.get(vertex, ()):
|
|
|
|
if neighbour in path or visit(neighbour):
|
|
|
|
return (vertex, neighbour)
|
|
|
|
path.remove(vertex)
|
2016-04-11 01:49:08 -04:00
|
|
|
|
2016-04-09 03:17:31 -04:00
|
|
|
for v in g:
|
|
|
|
cycle = visit(v)
|
|
|
|
if cycle:
|
|
|
|
return cycle
|
|
|
|
|
|
|
|
cycle = find_cycle(hierarchy)
|
|
|
|
if cycle:
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
2016-04-09 03:17:31 -04:00
|
|
|
"There's a loop in your Sankey, please provide a tree. "
|
2017-08-10 23:57:05 -04:00
|
|
|
"Here's a faulty link: {}").format(cycle))
|
2016-04-09 03:17:31 -04:00
|
|
|
return recs
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
class DirectedForceViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""An animated directed force layout graph visualization"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'directed_force'
|
|
|
|
verbose_name = _('Directed Force Layout')
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = 'd3noob @<a href="http://bl.ocks.org/d3noob/5141278">bl.ocks.org</a>'
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
|
|
|
qry = super(DirectedForceViz, self).query_obj()
|
|
|
|
if len(self.form_data['groupby']) != 2:
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_("Pick exactly 2 columns to 'Group By'"))
|
2016-03-18 02:44:58 -04:00
|
|
|
qry['metrics'] = [self.form_data['metric']]
|
|
|
|
return qry
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
df.columns = ['source', 'target', 'value']
|
2016-03-16 23:25:41 -04:00
|
|
|
return df.to_dict(orient='records')
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2017-06-26 19:44:47 -04:00
|
|
|
class ChordViz(BaseViz):
|
|
|
|
|
|
|
|
"""A Chord diagram"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'chord'
|
|
|
|
verbose_name = _('Directed Force Layout')
|
2017-06-26 19:44:47 -04:00
|
|
|
credits = '<a href="https://github.com/d3/d3-chord">Bostock</a>'
|
|
|
|
is_timeseries = False
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
qry = super(ChordViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
|
|
|
qry['groupby'] = [fd.get('groupby'), fd.get('columns')]
|
|
|
|
qry['metrics'] = [fd.get('metric')]
|
|
|
|
return qry
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
df.columns = ['source', 'target', 'value']
|
|
|
|
|
|
|
|
# Preparing a symetrical matrix like d3.chords calls for
|
|
|
|
nodes = list(set(df['source']) | set(df['target']))
|
|
|
|
matrix = {}
|
|
|
|
for source, target in product(nodes, nodes):
|
|
|
|
matrix[(source, target)] = 0
|
|
|
|
for source, target, value in df.to_records(index=False):
|
|
|
|
matrix[(source, target)] = value
|
|
|
|
m = [[matrix[(n1, n2)] for n1 in nodes] for n2 in nodes]
|
|
|
|
return {
|
|
|
|
'nodes': list(nodes),
|
|
|
|
'matrix': m,
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-05-23 13:43:33 -04:00
|
|
|
class CountryMapViz(BaseViz):
|
|
|
|
|
|
|
|
"""A country centric"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'country_map'
|
|
|
|
verbose_name = _('Country Map')
|
2017-05-23 13:43:33 -04:00
|
|
|
is_timeseries = False
|
|
|
|
credits = 'From bl.ocks.org By john-guerra'
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
qry = super(CountryMapViz, self).query_obj()
|
|
|
|
qry['metrics'] = [
|
|
|
|
self.form_data['metric']]
|
|
|
|
qry['groupby'] = [self.form_data['entity']]
|
|
|
|
return qry
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
cols = [fd.get('entity')]
|
|
|
|
metric = fd.get('metric')
|
|
|
|
cols += [metric]
|
|
|
|
ndf = df[cols]
|
|
|
|
df = ndf
|
|
|
|
df.columns = ['country_id', 'metric']
|
|
|
|
d = df.to_dict(orient='records')
|
|
|
|
return d
|
|
|
|
|
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
class WorldMapViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A country centric world map"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'world_map'
|
|
|
|
verbose_name = _('World Map')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = 'datamaps on <a href="https://www.npmjs.com/package/datamaps">npm</a>'
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
|
|
|
qry = super(WorldMapViz, self).query_obj()
|
|
|
|
qry['metrics'] = [
|
|
|
|
self.form_data['metric'], self.form_data['secondary_metric']]
|
|
|
|
qry['groupby'] = [self.form_data['entity']]
|
|
|
|
return qry
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-11-10 02:08:22 -05:00
|
|
|
from superset.data import countries
|
2017-03-04 00:20:25 -05:00
|
|
|
fd = self.form_data
|
|
|
|
cols = [fd.get('entity')]
|
|
|
|
metric = fd.get('metric')
|
|
|
|
secondary_metric = fd.get('secondary_metric')
|
2016-03-18 02:44:58 -04:00
|
|
|
if metric == secondary_metric:
|
|
|
|
ndf = df[cols]
|
2016-07-10 22:38:12 -04:00
|
|
|
# df[metric] will be a DataFrame
|
|
|
|
# because there are duplicate column names
|
|
|
|
ndf['m1'] = df[metric].iloc[:, 0]
|
|
|
|
ndf['m2'] = ndf['m1']
|
2016-03-18 02:44:58 -04:00
|
|
|
else:
|
|
|
|
cols += [metric, secondary_metric]
|
|
|
|
ndf = df[cols]
|
|
|
|
df = ndf
|
|
|
|
df.columns = ['country', 'm1', 'm2']
|
|
|
|
d = df.to_dict(orient='records')
|
|
|
|
for row in d:
|
2016-07-10 22:38:12 -04:00
|
|
|
country = None
|
2016-07-11 17:36:12 -04:00
|
|
|
if isinstance(row['country'], string_types):
|
2016-07-10 22:38:12 -04:00
|
|
|
country = countries.get(
|
2017-03-04 00:20:25 -05:00
|
|
|
fd.get('country_fieldtype'), row['country'])
|
2016-07-10 22:38:12 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
if country:
|
|
|
|
row['country'] = country['cca3']
|
|
|
|
row['latitude'] = country['lat']
|
|
|
|
row['longitude'] = country['lng']
|
|
|
|
row['name'] = country['name']
|
|
|
|
else:
|
2017-11-14 00:06:51 -05:00
|
|
|
row['country'] = 'XXX'
|
2016-03-16 23:25:41 -04:00
|
|
|
return d
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
|
|
|
class FilterBoxViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A multi filter, multi-choice filter box to make dashboards interactive"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'filter_box'
|
|
|
|
verbose_name = _('Filters')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-11-10 02:08:22 -05:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
2018-02-07 17:49:19 -05:00
|
|
|
cache_type = 'get_data'
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
2018-01-24 19:09:03 -05:00
|
|
|
return None
|
|
|
|
|
2018-02-07 17:49:19 -05:00
|
|
|
def run_extra_queries(self):
|
|
|
|
qry = self.filter_query_obj()
|
|
|
|
filters = [g for g in self.form_data['groupby']]
|
|
|
|
self.dataframes = {}
|
|
|
|
for flt in filters:
|
|
|
|
qry['groupby'] = [flt]
|
|
|
|
df = self.get_df_payload(query_obj=qry).get('df')
|
|
|
|
self.dataframes[flt] = df
|
|
|
|
|
2018-01-24 19:09:03 -05:00
|
|
|
def filter_query_obj(self):
|
2016-03-18 02:44:58 -04:00
|
|
|
qry = super(FilterBoxViz, self).query_obj()
|
2016-09-23 18:07:33 -04:00
|
|
|
groupby = self.form_data.get('groupby')
|
|
|
|
if len(groupby) < 1 and not self.form_data.get('date_filter'):
|
2017-11-14 00:06:51 -05:00
|
|
|
raise Exception(_('Pick at least one filter field'))
|
2016-03-18 02:44:58 -04:00
|
|
|
qry['metrics'] = [
|
|
|
|
self.form_data['metric']]
|
|
|
|
return qry
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
d = {}
|
2018-02-07 17:49:19 -05:00
|
|
|
filters = [g for g in self.form_data['groupby']]
|
2016-03-18 02:44:58 -04:00
|
|
|
for flt in filters:
|
2018-02-07 17:49:19 -05:00
|
|
|
df = self.dataframes[flt]
|
2016-03-16 23:25:41 -04:00
|
|
|
d[flt] = [{
|
|
|
|
'id': row[0],
|
2016-03-18 02:44:58 -04:00
|
|
|
'text': row[0],
|
|
|
|
'filter': flt,
|
|
|
|
'metric': row[1]}
|
2016-03-16 23:25:41 -04:00
|
|
|
for row in df.itertuples(index=False)
|
|
|
|
]
|
2016-03-18 02:44:58 -04:00
|
|
|
return d
|
|
|
|
|
|
|
|
|
|
|
|
class IFrameViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""You can squeeze just about anything in this iFrame component"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'iframe'
|
|
|
|
verbose_name = _('iFrame')
|
2016-11-10 02:08:22 -05:00
|
|
|
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
|
|
|
|
2018-01-17 16:54:10 -05:00
|
|
|
def query_obj(self):
|
|
|
|
return None
|
|
|
|
|
|
|
|
def get_df(self, query_obj=None):
|
2017-11-10 15:06:22 -05:00
|
|
|
return None
|
2017-04-13 14:37:27 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
class ParallelCoordinatesViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""Interactive parallel coordinate implementation
|
|
|
|
|
|
|
|
Uses this amazing javascript library
|
|
|
|
https://github.com/syntagmatic/parallel-coordinates
|
|
|
|
"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'para'
|
|
|
|
verbose_name = _('Parallel Coordinates')
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = (
|
|
|
|
'<a href="https://syntagmatic.github.io/parallel-coordinates/">'
|
2017-11-14 00:06:51 -05:00
|
|
|
"Syntagmatic's library</a>")
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
|
|
|
d = super(ParallelCoordinatesViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
2016-05-10 12:39:33 -04:00
|
|
|
d['metrics'] = copy.copy(fd.get('metrics'))
|
2016-03-18 02:44:58 -04:00
|
|
|
second = fd.get('secondary_metric')
|
|
|
|
if second not in d['metrics']:
|
|
|
|
d['metrics'] += [second]
|
|
|
|
d['groupby'] = [fd.get('series')]
|
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2017-11-14 00:06:51 -05:00
|
|
|
return df.to_dict(orient='records')
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
class HeatmapViz(BaseViz):
|
2016-03-16 23:25:41 -04:00
|
|
|
|
|
|
|
"""A nice heatmap visualization that support high density through canvas"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'heatmap'
|
|
|
|
verbose_name = _('Heatmap')
|
2016-03-18 02:44:58 -04:00
|
|
|
is_timeseries = False
|
2016-04-09 16:17:20 -04:00
|
|
|
credits = (
|
|
|
|
'inspired from mbostock @<a href="http://bl.ocks.org/mbostock/3074470">'
|
|
|
|
'bl.ocks.org</a>')
|
2016-03-16 23:25:41 -04:00
|
|
|
|
2016-03-18 02:44:58 -04:00
|
|
|
def query_obj(self):
|
|
|
|
d = super(HeatmapViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
|
|
|
d['metrics'] = [fd.get('metric')]
|
|
|
|
d['groupby'] = [fd.get('all_columns_x'), fd.get('all_columns_y')]
|
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2016-03-18 02:44:58 -04:00
|
|
|
fd = self.form_data
|
|
|
|
x = fd.get('all_columns_x')
|
|
|
|
y = fd.get('all_columns_y')
|
|
|
|
v = fd.get('metric')
|
|
|
|
if x == y:
|
|
|
|
df.columns = ['x', 'y', 'v']
|
|
|
|
else:
|
|
|
|
df = df[[x, y, v]]
|
|
|
|
df.columns = ['x', 'y', 'v']
|
|
|
|
norm = fd.get('normalize_across')
|
|
|
|
overall = False
|
2017-09-13 19:13:45 -04:00
|
|
|
max_ = df.v.max()
|
|
|
|
min_ = df.v.min()
|
|
|
|
bounds = fd.get('y_axis_bounds')
|
2017-09-19 00:32:39 -04:00
|
|
|
if bounds and bounds[0] is not None:
|
2017-09-13 19:13:45 -04:00
|
|
|
min_ = bounds[0]
|
2017-09-19 00:32:39 -04:00
|
|
|
if bounds and bounds[1] is not None:
|
2017-09-13 19:13:45 -04:00
|
|
|
max_ = bounds[1]
|
2016-03-18 02:44:58 -04:00
|
|
|
if norm == 'heatmap':
|
|
|
|
overall = True
|
|
|
|
else:
|
|
|
|
gb = df.groupby(norm, group_keys=False)
|
|
|
|
if len(gb) <= 1:
|
|
|
|
overall = True
|
|
|
|
else:
|
|
|
|
df['perc'] = (
|
|
|
|
gb.apply(
|
|
|
|
lambda x: (x.v - x.v.min()) / (x.v.max() - x.v.min()))
|
|
|
|
)
|
|
|
|
if overall:
|
2017-09-13 19:13:45 -04:00
|
|
|
df['perc'] = (df.v - min_) / (max_ - min_)
|
|
|
|
return {
|
2017-11-14 00:06:51 -05:00
|
|
|
'records': df.to_dict(orient='records'),
|
2017-09-13 19:13:45 -04:00
|
|
|
'extents': [min_, max_],
|
|
|
|
}
|
2016-03-18 02:44:58 -04:00
|
|
|
|
|
|
|
|
2016-05-17 01:49:12 -04:00
|
|
|
class HorizonViz(NVD3TimeSeriesViz):
|
|
|
|
|
|
|
|
"""Horizon chart
|
|
|
|
|
|
|
|
https://www.npmjs.com/package/d3-horizon-chart
|
|
|
|
"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'horizon'
|
|
|
|
verbose_name = _('Horizon Charts')
|
2016-05-17 01:49:12 -04:00
|
|
|
credits = (
|
|
|
|
'<a href="https://www.npmjs.com/package/d3-horizon-chart">'
|
|
|
|
'd3-horizon-chart</a>')
|
|
|
|
|
|
|
|
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
class MapboxViz(BaseViz):
|
|
|
|
|
|
|
|
"""Rich maps made with Mapbox"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'mapbox'
|
|
|
|
verbose_name = _('Mapbox')
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
is_timeseries = False
|
|
|
|
credits = (
|
|
|
|
'<a href=https://www.mapbox.com/mapbox-gl-js/api/>Mapbox GL JS</a>')
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(MapboxViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
|
|
|
label_col = fd.get('mapbox_label')
|
|
|
|
|
|
|
|
if not fd.get('groupby'):
|
|
|
|
d['columns'] = [fd.get('all_columns_x'), fd.get('all_columns_y')]
|
|
|
|
|
|
|
|
if label_col and len(label_col) >= 1:
|
2017-11-14 00:06:51 -05:00
|
|
|
if label_col[0] == 'count':
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
|
|
|
"Must have a [Group By] column to have 'count' as the [Label]"))
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
d['columns'].append(label_col[0])
|
|
|
|
|
|
|
|
if fd.get('point_radius') != 'Auto':
|
|
|
|
d['columns'].append(fd.get('point_radius'))
|
|
|
|
|
|
|
|
d['columns'] = list(set(d['columns']))
|
|
|
|
else:
|
|
|
|
# Ensuring columns chosen are all in group by
|
|
|
|
if (label_col and len(label_col) >= 1 and
|
2017-11-14 00:06:51 -05:00
|
|
|
label_col[0] != 'count' and
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
label_col[0] not in fd.get('groupby')):
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
2017-11-14 00:06:51 -05:00
|
|
|
'Choice of [Label] must be present in [Group By]'))
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
if (fd.get('point_radius') != 'Auto' and
|
|
|
|
fd.get('point_radius') not in fd.get('groupby')):
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
2017-11-14 00:06:51 -05:00
|
|
|
'Choice of [Point Radius] must be present in [Group By]'))
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
|
|
|
|
if (fd.get('all_columns_x') not in fd.get('groupby') or
|
|
|
|
fd.get('all_columns_y') not in fd.get('groupby')):
|
2017-08-10 23:57:05 -04:00
|
|
|
raise Exception(_(
|
2017-11-14 00:06:51 -05:00
|
|
|
'[Longitude] and [Latitude] columns must be present in [Group By]'))
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
return d
|
|
|
|
|
2017-02-16 20:28:35 -05:00
|
|
|
def get_data(self, df):
|
2018-02-20 17:41:35 -05:00
|
|
|
if df is None:
|
|
|
|
return None
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
fd = self.form_data
|
|
|
|
label_col = fd.get('mapbox_label')
|
|
|
|
custom_metric = label_col and len(label_col) >= 1
|
|
|
|
metric_col = [None] * len(df.index)
|
|
|
|
if custom_metric:
|
|
|
|
if label_col[0] == fd.get('all_columns_x'):
|
|
|
|
metric_col = df[fd.get('all_columns_x')]
|
|
|
|
elif label_col[0] == fd.get('all_columns_y'):
|
|
|
|
metric_col = df[fd.get('all_columns_y')]
|
|
|
|
else:
|
|
|
|
metric_col = df[label_col[0]]
|
|
|
|
point_radius_col = (
|
|
|
|
[None] * len(df.index)
|
2017-11-14 00:06:51 -05:00
|
|
|
if fd.get('point_radius') == 'Auto'
|
|
|
|
else df[fd.get('point_radius')])
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
|
|
|
|
# using geoJSON formatting
|
|
|
|
geo_json = {
|
2017-11-14 00:06:51 -05:00
|
|
|
'type': 'FeatureCollection',
|
|
|
|
'features': [
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
{
|
2017-11-14 00:06:51 -05:00
|
|
|
'type': 'Feature',
|
|
|
|
'properties': {
|
|
|
|
'metric': metric,
|
|
|
|
'radius': point_radius,
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
},
|
2017-11-14 00:06:51 -05:00
|
|
|
'geometry': {
|
|
|
|
'type': 'Point',
|
|
|
|
'coordinates': [lon, lat],
|
2017-11-08 00:32:45 -05:00
|
|
|
},
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
}
|
|
|
|
for lon, lat, metric, point_radius
|
|
|
|
in zip(
|
|
|
|
df[fd.get('all_columns_x')],
|
|
|
|
df[fd.get('all_columns_y')],
|
|
|
|
metric_col, point_radius_col)
|
2017-11-08 00:32:45 -05:00
|
|
|
],
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
return {
|
2017-11-14 00:06:51 -05:00
|
|
|
'geoJSON': geo_json,
|
|
|
|
'customMetric': custom_metric,
|
|
|
|
'mapboxApiKey': config.get('MAPBOX_API_KEY'),
|
|
|
|
'mapStyle': fd.get('mapbox_style'),
|
|
|
|
'aggregatorName': fd.get('pandas_aggfunc'),
|
|
|
|
'clusteringRadius': fd.get('clustering_radius'),
|
|
|
|
'pointRadiusUnit': fd.get('point_radius_unit'),
|
|
|
|
'globalOpacity': fd.get('global_opacity'),
|
|
|
|
'viewportLongitude': fd.get('viewport_longitude'),
|
|
|
|
'viewportLatitude': fd.get('viewport_latitude'),
|
|
|
|
'viewportZoom': fd.get('viewport_zoom'),
|
|
|
|
'renderWhileDragging': fd.get('render_while_dragging'),
|
|
|
|
'tooltip': fd.get('rich_tooltip'),
|
|
|
|
'color': fd.get('mapbox_color'),
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
}
|
|
|
|
|
2017-10-04 13:17:33 -04:00
|
|
|
|
2017-12-26 13:47:29 -05:00
|
|
|
class DeckGLMultiLayer(BaseViz):
|
|
|
|
|
|
|
|
"""Pile on multiple DeckGL layers"""
|
|
|
|
|
|
|
|
viz_type = 'deck_multi'
|
|
|
|
verbose_name = _('Deck.gl - Multiple Layers')
|
|
|
|
|
|
|
|
is_timeseries = False
|
|
|
|
credits = '<a href="https://uber.github.io/deck.gl/">deck.gl</a>'
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
return None
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
# Late imports to avoid circular import issues
|
|
|
|
from superset.models.core import Slice
|
|
|
|
from superset import db
|
|
|
|
slice_ids = fd.get('deck_slices')
|
|
|
|
slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all()
|
|
|
|
return {
|
|
|
|
'mapboxApiKey': config.get('MAPBOX_API_KEY'),
|
|
|
|
'slices': [slc.data for slc in slices],
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
class BaseDeckGLViz(BaseViz):
|
|
|
|
|
|
|
|
"""Base class for deck.gl visualizations"""
|
|
|
|
|
|
|
|
is_timeseries = False
|
|
|
|
credits = '<a href="https://uber.github.io/deck.gl/">deck.gl</a>'
|
2018-01-12 14:06:11 -05:00
|
|
|
spatial_control_keys = []
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
def get_metrics(self):
|
|
|
|
self.metric = self.form_data.get('size')
|
2017-12-15 14:47:27 -05:00
|
|
|
return [self.metric] if self.metric else []
|
2017-11-16 03:30:02 -05:00
|
|
|
|
2018-01-12 14:06:11 -05:00
|
|
|
def process_spatial_query_obj(self, key, group_by):
|
|
|
|
spatial = self.form_data.get(key)
|
|
|
|
if spatial is None:
|
|
|
|
raise ValueError(_('Bad spatial key'))
|
|
|
|
|
|
|
|
if spatial.get('type') == 'latlong':
|
|
|
|
group_by += [spatial.get('lonCol')]
|
|
|
|
group_by += [spatial.get('latCol')]
|
|
|
|
elif spatial.get('type') == 'delimited':
|
|
|
|
group_by += [spatial.get('lonlatCol')]
|
|
|
|
elif spatial.get('type') == 'geohash':
|
|
|
|
group_by += [spatial.get('geohashCol')]
|
|
|
|
|
|
|
|
def process_spatial_data_obj(self, key, df):
|
|
|
|
spatial = self.form_data.get(key)
|
|
|
|
if spatial is None:
|
|
|
|
raise ValueError(_('Bad spatial key'))
|
|
|
|
if spatial.get('type') == 'latlong':
|
2018-02-20 17:41:35 -05:00
|
|
|
df[key] = list(zip(
|
|
|
|
pd.to_numeric(df[spatial.get('lonCol')], errors='coerce'),
|
|
|
|
pd.to_numeric(df[spatial.get('latCol')], errors='coerce'),
|
|
|
|
))
|
2018-01-12 14:06:11 -05:00
|
|
|
elif spatial.get('type') == 'delimited':
|
2018-02-20 17:41:35 -05:00
|
|
|
|
|
|
|
def tupleify(s):
|
|
|
|
p = Point(s)
|
|
|
|
return (p.latitude, p.longitude)
|
|
|
|
|
|
|
|
df[key] = df[spatial.get('lonlatCol')].apply(tupleify)
|
|
|
|
|
2018-01-12 14:06:11 -05:00
|
|
|
if spatial.get('reverseCheckbox'):
|
2018-01-18 18:22:22 -05:00
|
|
|
df[key] = [
|
|
|
|
tuple(reversed(o)) if isinstance(o, (list, tuple)) else (0, 0)
|
|
|
|
for o in df[key]
|
|
|
|
]
|
2018-01-12 14:06:11 -05:00
|
|
|
del df[spatial.get('lonlatCol')]
|
|
|
|
elif spatial.get('type') == 'geohash':
|
|
|
|
latlong = df[spatial.get('geohashCol')].map(geohash.decode)
|
|
|
|
df[key] = list(zip(latlong.apply(lambda x: x[0]),
|
|
|
|
latlong.apply(lambda x: x[1])))
|
|
|
|
del df[spatial.get('geohashCol')]
|
|
|
|
return df
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(BaseDeckGLViz, self).query_obj()
|
|
|
|
fd = self.form_data
|
2017-12-15 14:47:27 -05:00
|
|
|
gb = []
|
|
|
|
|
2018-01-12 14:06:11 -05:00
|
|
|
for key in self.spatial_control_keys:
|
|
|
|
self.process_spatial_query_obj(key, gb)
|
2017-12-15 14:47:27 -05:00
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
if fd.get('dimension'):
|
2017-12-15 14:47:27 -05:00
|
|
|
gb += [fd.get('dimension')]
|
2017-11-16 03:30:02 -05:00
|
|
|
|
2018-01-11 18:42:44 -05:00
|
|
|
if fd.get('js_columns'):
|
|
|
|
gb += fd.get('js_columns')
|
2017-12-15 14:47:27 -05:00
|
|
|
metrics = self.get_metrics()
|
2018-04-11 18:15:36 -04:00
|
|
|
gb = list(set(gb))
|
2017-12-15 14:47:27 -05:00
|
|
|
if metrics:
|
|
|
|
d['groupby'] = gb
|
|
|
|
d['metrics'] = self.get_metrics()
|
|
|
|
else:
|
|
|
|
d['columns'] = gb
|
2018-01-12 14:06:11 -05:00
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
return d
|
|
|
|
|
2018-01-11 18:42:44 -05:00
|
|
|
def get_js_columns(self, d):
|
|
|
|
cols = self.form_data.get('js_columns') or []
|
|
|
|
return {col: d.get(col) for col in cols}
|
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
def get_data(self, df):
|
2018-02-20 17:41:35 -05:00
|
|
|
if df is None:
|
|
|
|
return None
|
2018-01-12 14:06:11 -05:00
|
|
|
for key in self.spatial_control_keys:
|
|
|
|
df = self.process_spatial_data_obj(key, df)
|
2017-12-15 14:47:27 -05:00
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
features = []
|
|
|
|
for d in df.to_dict(orient='records'):
|
2018-01-17 17:01:32 -05:00
|
|
|
feature = self.get_properties(d)
|
|
|
|
extra_props = self.get_js_columns(d)
|
|
|
|
if extra_props:
|
|
|
|
feature['extraProps'] = extra_props
|
2018-01-11 18:42:44 -05:00
|
|
|
features.append(feature)
|
2018-01-18 16:28:46 -05:00
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
return {
|
2017-11-16 14:18:33 -05:00
|
|
|
'features': features,
|
|
|
|
'mapboxApiKey': config.get('MAPBOX_API_KEY'),
|
2017-11-16 03:30:02 -05:00
|
|
|
}
|
|
|
|
|
2018-01-17 17:01:32 -05:00
|
|
|
def get_properties(self, d):
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
class DeckScatterViz(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's ScatterLayer"""
|
|
|
|
|
2017-11-16 14:18:33 -05:00
|
|
|
viz_type = 'deck_scatter'
|
|
|
|
verbose_name = _('Deck.gl - Scatter plot')
|
2018-01-12 14:06:11 -05:00
|
|
|
spatial_control_keys = ['spatial']
|
2018-02-15 20:55:11 -05:00
|
|
|
is_timeseries = True
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
def query_obj(self):
|
2017-12-15 14:47:27 -05:00
|
|
|
fd = self.form_data
|
2018-02-15 20:55:11 -05:00
|
|
|
self.is_timeseries = fd.get('time_grain_sqla') or fd.get('granularity')
|
2017-12-15 14:47:27 -05:00
|
|
|
self.point_radius_fixed = (
|
|
|
|
fd.get('point_radius_fixed') or {'type': 'fix', 'value': 500})
|
2017-11-16 03:30:02 -05:00
|
|
|
return super(DeckScatterViz, self).query_obj()
|
|
|
|
|
|
|
|
def get_metrics(self):
|
2017-12-19 15:38:03 -05:00
|
|
|
self.metric = None
|
2017-11-16 03:30:02 -05:00
|
|
|
if self.point_radius_fixed.get('type') == 'metric':
|
2017-12-19 15:38:03 -05:00
|
|
|
self.metric = self.point_radius_fixed.get('value')
|
|
|
|
return [self.metric]
|
2017-12-15 14:47:27 -05:00
|
|
|
return None
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
def get_properties(self, d):
|
|
|
|
return {
|
2018-04-02 20:48:56 -04:00
|
|
|
'metric': d.get(self.metric),
|
2017-11-16 14:18:33 -05:00
|
|
|
'radius': self.fixed_value if self.fixed_value else d.get(self.metric),
|
|
|
|
'cat_color': d.get(self.dim) if self.dim else None,
|
2018-01-17 17:01:32 -05:00
|
|
|
'position': d.get('spatial'),
|
2018-03-09 18:28:11 -05:00
|
|
|
'__timestamp': d.get(DTTM_ALIAS) or d.get('__time'),
|
2017-11-16 03:30:02 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
self.point_radius_fixed = fd.get('point_radius_fixed')
|
|
|
|
self.fixed_value = None
|
|
|
|
self.dim = self.form_data.get('dimension')
|
|
|
|
if self.point_radius_fixed.get('type') != 'metric':
|
|
|
|
self.fixed_value = self.point_radius_fixed.get('value')
|
|
|
|
return super(DeckScatterViz, self).get_data(df)
|
|
|
|
|
|
|
|
|
|
|
|
class DeckScreengrid(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's ScreenGridLayer"""
|
|
|
|
|
2017-11-16 14:18:33 -05:00
|
|
|
viz_type = 'deck_screengrid'
|
|
|
|
verbose_name = _('Deck.gl - Screen Grid')
|
2018-01-12 14:06:11 -05:00
|
|
|
spatial_control_keys = ['spatial']
|
2018-04-09 17:02:20 -04:00
|
|
|
is_timeseries = True
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
fd = self.form_data
|
|
|
|
self.is_timeseries = fd.get('time_grain_sqla') or fd.get('granularity')
|
|
|
|
return super(DeckScreengrid, self).query_obj()
|
2018-01-12 14:06:11 -05:00
|
|
|
|
2018-01-17 17:01:32 -05:00
|
|
|
def get_properties(self, d):
|
|
|
|
return {
|
|
|
|
'position': d.get('spatial'),
|
|
|
|
'weight': d.get(self.metric) or 1,
|
2018-04-09 17:02:20 -04:00
|
|
|
'__timestamp': d.get(DTTM_ALIAS) or d.get('__time'),
|
2018-01-17 17:01:32 -05:00
|
|
|
}
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
|
|
|
|
class DeckGrid(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's DeckLayer"""
|
|
|
|
|
2017-11-16 14:18:33 -05:00
|
|
|
viz_type = 'deck_grid'
|
|
|
|
verbose_name = _('Deck.gl - 3D Grid')
|
2018-01-12 14:06:11 -05:00
|
|
|
spatial_control_keys = ['spatial']
|
|
|
|
|
2018-01-17 17:01:32 -05:00
|
|
|
def get_properties(self, d):
|
|
|
|
return {
|
|
|
|
'position': d.get('spatial'),
|
|
|
|
'weight': d.get(self.metric) or 1,
|
|
|
|
}
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
|
2017-12-19 15:38:03 -05:00
|
|
|
class DeckPathViz(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's PathLayer"""
|
|
|
|
|
|
|
|
viz_type = 'deck_path'
|
|
|
|
verbose_name = _('Deck.gl - Paths')
|
2018-01-18 16:28:46 -05:00
|
|
|
deck_viz_key = 'path'
|
2017-12-19 15:38:03 -05:00
|
|
|
deser_map = {
|
|
|
|
'json': json.loads,
|
|
|
|
'polyline': polyline.decode,
|
|
|
|
}
|
2018-01-12 14:06:11 -05:00
|
|
|
|
2017-12-19 15:38:03 -05:00
|
|
|
def query_obj(self):
|
|
|
|
d = super(DeckPathViz, self).query_obj()
|
2018-01-11 18:42:44 -05:00
|
|
|
line_col = self.form_data.get('line_column')
|
|
|
|
if d['metrics']:
|
|
|
|
d['groupby'].append(line_col)
|
|
|
|
else:
|
|
|
|
d['columns'].append(line_col)
|
2017-12-19 15:38:03 -05:00
|
|
|
return d
|
|
|
|
|
2018-01-11 18:42:44 -05:00
|
|
|
def get_properties(self, d):
|
2017-12-19 15:38:03 -05:00
|
|
|
fd = self.form_data
|
|
|
|
deser = self.deser_map[fd.get('line_type')]
|
2018-01-11 18:42:44 -05:00
|
|
|
path = deser(d[fd.get('line_column')])
|
2017-12-19 15:38:03 -05:00
|
|
|
if fd.get('reverse_long_lat'):
|
2018-01-11 18:42:44 -05:00
|
|
|
path = (path[1], path[0])
|
|
|
|
return {
|
2018-01-18 16:28:46 -05:00
|
|
|
self.deck_viz_key: path,
|
2017-12-19 15:38:03 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-01-18 16:28:46 -05:00
|
|
|
class DeckPolygon(DeckPathViz):
|
|
|
|
|
|
|
|
"""deck.gl's Polygon Layer"""
|
|
|
|
|
|
|
|
viz_type = 'deck_polygon'
|
|
|
|
deck_viz_key = 'polygon'
|
|
|
|
verbose_name = _('Deck.gl - Polygon')
|
|
|
|
|
|
|
|
|
2017-11-16 03:30:02 -05:00
|
|
|
class DeckHex(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's DeckLayer"""
|
|
|
|
|
2017-11-16 14:18:33 -05:00
|
|
|
viz_type = 'deck_hex'
|
|
|
|
verbose_name = _('Deck.gl - 3D HEX')
|
2018-01-12 14:06:11 -05:00
|
|
|
spatial_control_keys = ['spatial']
|
|
|
|
|
2018-01-17 17:01:32 -05:00
|
|
|
def get_properties(self, d):
|
|
|
|
return {
|
|
|
|
'position': d.get('spatial'),
|
|
|
|
'weight': d.get(self.metric) or 1,
|
|
|
|
}
|
2017-11-16 03:30:02 -05:00
|
|
|
|
|
|
|
|
2017-12-22 17:40:08 -05:00
|
|
|
class DeckGeoJson(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's GeoJSONLayer"""
|
|
|
|
|
|
|
|
viz_type = 'deck_geojson'
|
|
|
|
verbose_name = _('Deck.gl - GeoJSON')
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
d = super(DeckGeoJson, self).query_obj()
|
2018-01-17 17:01:32 -05:00
|
|
|
d['columns'] += [self.form_data.get('geojson')]
|
2017-12-22 17:40:08 -05:00
|
|
|
d['metrics'] = []
|
|
|
|
d['groupby'] = []
|
|
|
|
return d
|
|
|
|
|
2018-01-17 17:01:32 -05:00
|
|
|
def get_properties(self, d):
|
|
|
|
geojson = d.get(self.form_data.get('geojson'))
|
|
|
|
return json.loads(geojson)
|
2017-12-22 17:40:08 -05:00
|
|
|
|
|
|
|
|
2018-01-12 14:06:11 -05:00
|
|
|
class DeckArc(BaseDeckGLViz):
|
|
|
|
|
|
|
|
"""deck.gl's Arc Layer"""
|
|
|
|
|
|
|
|
viz_type = 'deck_arc'
|
|
|
|
verbose_name = _('Deck.gl - Arc')
|
|
|
|
spatial_control_keys = ['start_spatial', 'end_spatial']
|
|
|
|
|
2018-01-17 17:01:32 -05:00
|
|
|
def get_properties(self, d):
|
|
|
|
return {
|
|
|
|
'sourcePosition': d.get('start_spatial'),
|
|
|
|
'targetPosition': d.get('end_spatial'),
|
2018-01-12 14:06:11 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
d = super(DeckArc, self).get_data(df)
|
|
|
|
arcs = d['features']
|
|
|
|
|
|
|
|
return {
|
2018-01-25 17:07:34 -05:00
|
|
|
'arcs': arcs,
|
2018-01-12 14:06:11 -05:00
|
|
|
'mapboxApiKey': config.get('MAPBOX_API_KEY'),
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-07-21 19:29:25 -04:00
|
|
|
class EventFlowViz(BaseViz):
|
2017-11-16 03:30:02 -05:00
|
|
|
|
2017-07-21 19:29:25 -04:00
|
|
|
"""A visualization to explore patterns in event sequences"""
|
|
|
|
|
2017-11-14 00:06:51 -05:00
|
|
|
viz_type = 'event_flow'
|
|
|
|
verbose_name = _('Event flow')
|
2017-07-21 19:29:25 -04:00
|
|
|
credits = 'from <a href="https://github.com/williaster/data-ui">@data-ui</a>'
|
|
|
|
is_timeseries = True
|
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
query = super(EventFlowViz, self).query_obj()
|
|
|
|
form_data = self.form_data
|
|
|
|
|
|
|
|
event_key = form_data.get('all_columns_x')
|
|
|
|
entity_key = form_data.get('entity')
|
|
|
|
meta_keys = [
|
2017-10-04 13:17:33 -04:00
|
|
|
col for col in form_data.get('all_columns')
|
|
|
|
if col != event_key and col != entity_key
|
2017-07-21 19:29:25 -04:00
|
|
|
]
|
|
|
|
|
|
|
|
query['columns'] = [event_key, entity_key] + meta_keys
|
|
|
|
|
|
|
|
if form_data['order_by_entity']:
|
|
|
|
query['orderby'] = [(entity_key, True)]
|
|
|
|
|
|
|
|
return query
|
|
|
|
|
|
|
|
def get_data(self, df):
|
2017-11-14 00:06:51 -05:00
|
|
|
return df.to_dict(orient='records')
|
2017-07-21 19:29:25 -04:00
|
|
|
|
|
|
|
|
2017-09-26 18:11:35 -04:00
|
|
|
class PairedTTestViz(BaseViz):
|
|
|
|
|
|
|
|
"""A table displaying paired t-test values"""
|
|
|
|
|
|
|
|
viz_type = 'paired_ttest'
|
2017-11-14 00:06:51 -05:00
|
|
|
verbose_name = _('Time Series - Paired t-test')
|
2017-09-26 18:11:35 -04:00
|
|
|
sort_series = False
|
|
|
|
is_timeseries = True
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
"""
|
|
|
|
Transform received data frame into an object of the form:
|
|
|
|
{
|
2017-11-14 00:06:51 -05:00
|
|
|
'metric1': [
|
2017-09-26 18:11:35 -04:00
|
|
|
{
|
|
|
|
groups: ('groupA', ... ),
|
|
|
|
values: [ {x, y}, ... ],
|
|
|
|
}, ...
|
|
|
|
], ...
|
|
|
|
}
|
|
|
|
"""
|
|
|
|
fd = self.form_data
|
|
|
|
groups = fd.get('groupby')
|
|
|
|
metrics = fd.get('metrics')
|
|
|
|
df.fillna(0)
|
|
|
|
df = df.pivot_table(
|
|
|
|
index=DTTM_ALIAS,
|
|
|
|
columns=groups,
|
|
|
|
values=metrics)
|
|
|
|
cols = []
|
|
|
|
# Be rid of falsey keys
|
|
|
|
for col in df.columns:
|
|
|
|
if col == '':
|
|
|
|
cols.append('N/A')
|
|
|
|
elif col is None:
|
|
|
|
cols.append('NULL')
|
|
|
|
else:
|
|
|
|
cols.append(col)
|
|
|
|
df.columns = cols
|
|
|
|
data = {}
|
|
|
|
series = df.to_dict('series')
|
|
|
|
for nameSet in df.columns:
|
|
|
|
# If no groups are defined, nameSet will be the metric name
|
|
|
|
hasGroup = not isinstance(nameSet, string_types)
|
|
|
|
Y = series[nameSet]
|
|
|
|
d = {
|
|
|
|
'group': nameSet[1:] if hasGroup else 'All',
|
|
|
|
'values': [
|
|
|
|
{'x': t, 'y': Y[t] if t in Y else None}
|
|
|
|
for t in df.index
|
|
|
|
],
|
|
|
|
}
|
|
|
|
key = nameSet[0] if hasGroup else nameSet
|
|
|
|
if key in data:
|
|
|
|
data[key].append(d)
|
|
|
|
else:
|
|
|
|
data[key] = [d]
|
|
|
|
return data
|
|
|
|
|
Map visualization (#650)
* simple mapbox viz
use react-map-gl
superclustering of long/lat points
Added hook for map style, huge performance boost from bounding box fix, added count text on clusters
variable gradient size based on metric count
Ability to aggregate over any point property
This needed a change in the supercluster npm module, a PR was placed here:
https://github.com/mapbox/supercluster/pull/12
Aggregator function option in explore, tweaked visual defaults
better radius size management
clustering radius, point metric/unit options
scale cluster labels that don't fit, non-numeric labels for points
Minor fixes, label field affects points, text changes
serve mapbox apikey for slice
global opacity, viewport saves (hacky), bug in point labels
fixing mapbox-gl dependency
mapbox_api_key in config
expose row_limit, fix minor bugs
Add renderWhileDragging flag, groupby. Only show numerical columns for point radius
Implicitly group by lng/lat columns and error when label doesn't match groupby
'Fix' radius in miles problem, still some jankiness
derived fields cannot be typed as of now -> reverting numerical number change
better grouping error checking, expose count(*) for labelling
Custom colour for clusters/points + smart text colouring
Fixed bad positioning and overflow in explore view + small bugs + added thumbnail
* landscaping & eslint & use izip
* landscapin'
* address js code review
2016-06-24 17:16:51 -04:00
|
|
|
|
2018-02-03 23:18:24 -05:00
|
|
|
class RoseViz(NVD3TimeSeriesViz):
|
|
|
|
|
|
|
|
viz_type = 'rose'
|
|
|
|
verbose_name = _('Time Series - Nightingale Rose Chart')
|
|
|
|
sort_series = False
|
|
|
|
is_timeseries = True
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
data = super(RoseViz, self).get_data(df)
|
|
|
|
result = {}
|
|
|
|
for datum in data:
|
|
|
|
key = datum['key']
|
|
|
|
for val in datum['values']:
|
|
|
|
timestamp = val['x'].value
|
|
|
|
if not result.get(timestamp):
|
|
|
|
result[timestamp] = []
|
|
|
|
value = 0 if math.isnan(val['y']) else val['y']
|
|
|
|
result[timestamp].append({
|
|
|
|
'key': key,
|
|
|
|
'value': value,
|
|
|
|
'name': ', '.join(key) if isinstance(key, list) else key,
|
|
|
|
'time': val['x'],
|
|
|
|
})
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
2017-10-13 00:54:59 -04:00
|
|
|
class PartitionViz(NVD3TimeSeriesViz):
|
|
|
|
|
|
|
|
"""
|
|
|
|
A hierarchical data visualization with support for time series.
|
|
|
|
"""
|
|
|
|
|
|
|
|
viz_type = 'partition'
|
2017-11-14 00:06:51 -05:00
|
|
|
verbose_name = _('Partition Diagram')
|
2017-10-13 00:54:59 -04:00
|
|
|
|
|
|
|
def query_obj(self):
|
|
|
|
query_obj = super(PartitionViz, self).query_obj()
|
|
|
|
time_op = self.form_data.get('time_series_option', 'not_time')
|
|
|
|
# Return time series data if the user specifies so
|
|
|
|
query_obj['is_timeseries'] = time_op != 'not_time'
|
|
|
|
return query_obj
|
|
|
|
|
|
|
|
def levels_for(self, time_op, groups, df):
|
|
|
|
"""
|
|
|
|
Compute the partition at each `level` from the dataframe.
|
|
|
|
"""
|
|
|
|
levels = {}
|
|
|
|
for i in range(0, len(groups) + 1):
|
|
|
|
agg_df = df.groupby(groups[:i]) if i else df
|
|
|
|
levels[i] = (
|
|
|
|
agg_df.mean() if time_op == 'agg_mean'
|
|
|
|
else agg_df.sum(numeric_only=True))
|
|
|
|
return levels
|
|
|
|
|
|
|
|
def levels_for_diff(self, time_op, groups, df):
|
|
|
|
# Obtain a unique list of the time grains
|
|
|
|
times = list(set(df[DTTM_ALIAS]))
|
|
|
|
times.sort()
|
|
|
|
until = times[len(times) - 1]
|
|
|
|
since = times[0]
|
|
|
|
# Function describing how to calculate the difference
|
|
|
|
func = {
|
|
|
|
'point_diff': [
|
|
|
|
pd.Series.sub,
|
|
|
|
lambda a, b, fill_value: a - b,
|
|
|
|
],
|
|
|
|
'point_factor': [
|
|
|
|
pd.Series.div,
|
|
|
|
lambda a, b, fill_value: a / float(b),
|
|
|
|
],
|
|
|
|
'point_percent': [
|
|
|
|
lambda a, b, fill_value=0: a.div(b, fill_value=fill_value) - 1,
|
|
|
|
lambda a, b, fill_value: a / float(b) - 1,
|
|
|
|
],
|
|
|
|
}[time_op]
|
|
|
|
agg_df = df.groupby(DTTM_ALIAS).sum()
|
|
|
|
levels = {0: pd.Series({
|
|
|
|
m: func[1](agg_df[m][until], agg_df[m][since], 0)
|
|
|
|
for m in agg_df.columns})}
|
|
|
|
for i in range(1, len(groups) + 1):
|
|
|
|
agg_df = df.groupby([DTTM_ALIAS] + groups[:i]).sum()
|
|
|
|
levels[i] = pd.DataFrame({
|
|
|
|
m: func[0](agg_df[m][until], agg_df[m][since], fill_value=0)
|
|
|
|
for m in agg_df.columns})
|
|
|
|
return levels
|
|
|
|
|
|
|
|
def levels_for_time(self, groups, df):
|
|
|
|
procs = {}
|
|
|
|
for i in range(0, len(groups) + 1):
|
|
|
|
self.form_data['groupby'] = groups[:i]
|
|
|
|
df_drop = df.drop(groups[i:], 1)
|
|
|
|
procs[i] = self.process_data(df_drop, aggregate=True).fillna(0)
|
|
|
|
self.form_data['groupby'] = groups
|
|
|
|
return procs
|
|
|
|
|
|
|
|
def nest_values(self, levels, level=0, metric=None, dims=()):
|
|
|
|
"""
|
|
|
|
Nest values at each level on the back-end with
|
|
|
|
access and setting, instead of summing from the bottom.
|
|
|
|
"""
|
|
|
|
if not level:
|
|
|
|
return [{
|
|
|
|
'name': m,
|
|
|
|
'val': levels[0][m],
|
|
|
|
'children': self.nest_values(levels, 1, m),
|
|
|
|
} for m in levels[0].index]
|
|
|
|
if level == 1:
|
|
|
|
return [{
|
|
|
|
'name': i,
|
|
|
|
'val': levels[1][metric][i],
|
|
|
|
'children': self.nest_values(levels, 2, metric, (i,)),
|
|
|
|
} for i in levels[1][metric].index]
|
|
|
|
if level >= len(levels):
|
|
|
|
return []
|
|
|
|
return [{
|
|
|
|
'name': i,
|
|
|
|
'val': levels[level][metric][dims][i],
|
|
|
|
'children': self.nest_values(
|
2017-11-08 00:32:45 -05:00
|
|
|
levels, level + 1, metric, dims + (i,),
|
2017-10-13 00:54:59 -04:00
|
|
|
),
|
|
|
|
} for i in levels[level][metric][dims].index]
|
|
|
|
|
|
|
|
def nest_procs(self, procs, level=-1, dims=(), time=None):
|
|
|
|
if level == -1:
|
|
|
|
return [{
|
|
|
|
'name': m,
|
|
|
|
'children': self.nest_procs(procs, 0, (m,)),
|
|
|
|
} for m in procs[0].columns]
|
|
|
|
if not level:
|
|
|
|
return [{
|
|
|
|
'name': t,
|
|
|
|
'val': procs[0][dims[0]][t],
|
|
|
|
'children': self.nest_procs(procs, 1, dims, t),
|
|
|
|
} for t in procs[0].index]
|
|
|
|
if level >= len(procs):
|
|
|
|
return []
|
|
|
|
return [{
|
|
|
|
'name': i,
|
|
|
|
'val': procs[level][dims][i][time],
|
2017-11-08 00:32:45 -05:00
|
|
|
'children': self.nest_procs(procs, level + 1, dims + (i,), time),
|
2017-10-13 00:54:59 -04:00
|
|
|
} for i in procs[level][dims].columns]
|
|
|
|
|
|
|
|
def get_data(self, df):
|
|
|
|
fd = self.form_data
|
|
|
|
groups = fd.get('groupby', [])
|
|
|
|
time_op = fd.get('time_series_option', 'not_time')
|
|
|
|
if not len(groups):
|
|
|
|
raise ValueError('Please choose at least one groupby')
|
|
|
|
if time_op == 'not_time':
|
|
|
|
levels = self.levels_for('agg_sum', groups, df)
|
|
|
|
elif time_op in ['agg_sum', 'agg_mean']:
|
|
|
|
levels = self.levels_for(time_op, groups, df)
|
|
|
|
elif time_op in ['point_diff', 'point_factor', 'point_percent']:
|
|
|
|
levels = self.levels_for_diff(time_op, groups, df)
|
|
|
|
elif time_op == 'adv_anal':
|
|
|
|
procs = self.levels_for_time(groups, df)
|
|
|
|
return self.nest_procs(procs)
|
|
|
|
else:
|
|
|
|
levels = self.levels_for('agg_sum', [DTTM_ALIAS] + groups, df)
|
|
|
|
return self.nest_values(levels)
|
|
|
|
|
|
|
|
|
2017-10-04 13:17:33 -04:00
|
|
|
viz_types = {
|
|
|
|
o.viz_type: o for o in globals().values()
|
|
|
|
if (
|
|
|
|
inspect.isclass(o) and
|
|
|
|
issubclass(o, BaseViz) and
|
|
|
|
o.viz_type not in config.get('VIZ_TYPE_BLACKLIST'))}
|