Mostly working

This commit is contained in:
Maxime Beauchemin 2015-09-29 21:41:49 -07:00
parent f3f807d042
commit 74e5a3e868
8 changed files with 252 additions and 192 deletions

View File

@ -61,6 +61,8 @@ def load_examples(sample):
Column("num", Integer),
Column("ds", String(20)),
Column("gender", String(10)),
Column("sum_boys", Integer),
Column("sum_girls", Integer),
)
try:
BirthNames.drop(db.engine)
@ -82,7 +84,10 @@ def load_examples(sample):
state=state,
year=year,
ds=ds,
name=name, num=num, gender=gender)
name=name, num=num, gender=gender,
sum_boys=num if gender == 'boy' else 0,
sum_girls=num if gender == 'girl' else 0,
)
if i % 5000 == 0:
print("{} loaded out of 82527 rows".format(i))
session.commit()
@ -109,8 +114,11 @@ def load_examples(sample):
obj.main_dttm_col = 'ds'
obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=one+day&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table"
obj.database = dbobj
obj.columns = [models.TableColumn(
column_name="num", sum=True, type="INTEGER")]
obj.columns = [
models.TableColumn(column_name="num", sum=True, type="INTEGER"),
models.TableColumn(column_name="sum_boys", sum=True, type="INTEGER"),
models.TableColumn(column_name="sum_girls", sum=True, type="INTEGER"),
]
models.Table
session.add(obj)
session.commit()
@ -200,7 +208,7 @@ def load_examples(sample):
session.add(slc)
slices.append(slc)
slice_name = "States"
slice_name = "Gender by State"
slc = session.query(Slice).filter_by(slice_name=slice_name).first()
if not slc:
slc = Slice(
@ -210,6 +218,7 @@ def load_examples(sample):
table=tbl,
params=get_slice_json(
slice_name, flt_eq_1="other", viz_type="dist_bar",
metrics=['sum__sum_girls', 'sum__sum_boys'],
groupby=['state'], flt_op_1='not in', flt_col_1='state'))
session.add(slc)
slices.append(slc)

View File

@ -1,7 +1,9 @@
from wtforms import (
Field, Form, SelectMultipleField, SelectField, TextField, TextAreaField,
BooleanField, IntegerField)
BooleanField, IntegerField, HiddenField)
from copy import copy
from panoramix import app
config = app.config
class OmgWtForm(Form):
@ -27,157 +29,183 @@ class OmgWtForm(Form):
return ""
def form_factory(viz):
datasource = viz.datasource
from panoramix.viz import viz_types
row_limits = [10, 50, 100, 500, 1000, 5000, 10000]
class FormFactory(object):
row_limits = [10, 50, 100, 500, 1000, 5000, 10000, 50000]
series_limits = [0, 5, 10, 25, 50, 100, 500]
group_by_choices = [(s, s) for s in datasource.groupby_column_names]
# Pool of all the fields that can be used in Panoramix
px_form_fields = {
'viz_type': SelectField(
'Viz',
choices=[(k, v.verbose_name) for k, v in viz_types.items()],
description="The type of visualization to display"),
'metrics': SelectMultipleField(
'Metrics', choices=datasource.metrics_combo,
description="One or many metrics to display"),
'metric': SelectField(
'Metric', choices=datasource.metrics_combo,
description="One or many metrics to display"),
'groupby': SelectMultipleField(
'Group by',
choices=[(s, s) for s in datasource.groupby_column_names],
description="One or many fields to group by"),
'granularity': TextField(
'Time Granularity', default="one day",
description=(
"The time granularity for the visualization. Note that you "
"can type and use simple natural language as in '10 seconds', "
"'1 day' or '56 weeks'")),
'since': TextField(
'Since', default="one day ago", description=(
"Timestamp from filter. This supports free form typing and "
"natural language as in '1 day ago', '28 days' or '3 years'")),
'until': TextField('Until', default="now"),
'row_limit':
SelectField(
'Row limit', choices=[(s, s) for s in row_limits]),
'limit':
SelectField(
'Series limit', choices=[(s, s) for s in series_limits],
def __init__(self, viz):
self.viz = viz
from panoramix.viz import viz_types
viz = self.viz
datasource = viz.datasource
group_by_choices = [(s, s) for s in datasource.groupby_column_names]
# Pool of all the fields that can be used in Panoramix
self.field_dict = {
'viz_type': SelectField(
'Viz',
default='table',
choices=[(k, v.verbose_name) for k, v in viz_types.items()],
description="The type of visualization to display"),
'metrics': SelectMultipleField(
'Metrics', choices=datasource.metrics_combo,
default=[datasource.metrics_combo[0][0]],
description="One or many metrics to display"),
'metric': SelectField(
'Metric', choices=datasource.metrics_combo,
description="One or many metrics to display"),
'groupby': SelectMultipleField(
'Group by',
choices=[(s, s) for s in datasource.groupby_column_names],
description="One or many fields to group by"),
'granularity': TextField(
'Time Granularity', default="one day",
description=(
"Limits the number of time series that get displayed")),
'rolling_type': SelectField(
'Rolling',
choices=[(s, s) for s in ['mean', 'sum', 'std']],
description=(
"Defines a rolling window function to apply")),
'rolling_periods': TextField('Periods', description=(
"Defines the size of the rolling window function, "
"relative to the 'granularity' field")),
'series': SelectField(
'Series', choices=group_by_choices,
description=(
"Defines the grouping of entities. "
"Each serie is shown as a specific color on the chart and "
"has a legend toggle")),
'entity': SelectField('Entity', choices=group_by_choices,
description="This define the element to be plotted on the chart"),
'x': SelectField(
'X Axis', choices=datasource.metrics_combo,
description="Metric assigned to the [X] axis"),
'y': SelectField('Y Axis', choices=datasource.metrics_combo,
description="Metric assigned to the [Y] axis"),
'size': SelectField('Bubble Size', choices=datasource.metrics_combo),
'where': TextField('Custom WHERE clause'),
'compare_lag': TextField('Comparison Period Lag',
description="Based on granularity, number of time periods to compare against"),
'compare_suffix': TextField('Comparison suffix',
description="Suffix to apply after the percentage display"),
'markup_type': SelectField(
"Markup Type",
choices=[(s, s) for s in ['markdown', 'html']],
default="markdown",
description="Pick your favorite markup language"),
'rotation': SelectField(
"Rotation",
choices=[(s, s) for s in ['random', 'flat', 'square']],
default="random",
description="Rotation to apply to words in the cloud"),
'code': TextAreaField("Code", description="Put your code here"),
'size_from': TextField(
"Font Size From",
default="20",
description="Font size for the smallest value in the list"),
'size_to': TextField(
"Font Size To",
default="150",
description="Font size for the biggest value in the list"),
'show_brush': BooleanField(
"Range Selector", default=True,
description="Whether to display the time range interactive selector"),
'show_legend': BooleanField(
"Legend", default=True, false_values=["f"],
description="Whether to display the legend (toggles)"),
'rich_tooltip': BooleanField(
"Rich Tooltip", default=True,
description="The rich tooltip shows a list of all series for that point in time"),
'y_axis_zero': BooleanField(
"Y Axis Zero", default=False,
description="Force the Y axis to start at 0 instead of the minimum value"),
'y_log_scale': BooleanField(
"Y Log", default=False,
description="Use a log scale for the Y axis"),
'x_log_scale': BooleanField(
"X Log", default=False,
description="Use a log scale for the X axis"),
'donut': BooleanField(
"Donut", default=False,
description="Do you want a donut or a pie?"),
'contribution': BooleanField(
"Contribution", default=False,
description="Compute the contribution to the total"),
'num_period_compare': IntegerField(
"Period Ratio", default=None,
description=(
"Number of period to compare against, "
"this is relative to the granularity selected")),
}
field_css_classes = {k: ['form-control'] for k in px_form_fields.keys()}
select2 = [
'viz_type', 'metrics', 'groupby',
'row_limit', 'rolling_type', 'series',
'entity', 'x', 'y', 'size', 'rotation', 'metric', 'limit',
'markup_type',]
field_css_classes['since'] += ['select2_free_since']
field_css_classes['until'] += ['select2_free_until']
field_css_classes['granularity'] += ['select2_free_granularity']
for field in ('show_brush', 'show_legend', 'rich_tooltip'):
field_css_classes[field] += ['input-sm']
for field in select2:
field_css_classes[field] += ['select2']
"The time granularity for the visualization. Note that you "
"can type and use simple natural language as in '10 seconds', "
"'1 day' or '56 weeks'")),
'since': TextField(
'Since', default="one day ago", description=(
"Timestamp from filter. This supports free form typing and "
"natural language as in '1 day ago', '28 days' or '3 years'")),
'until': TextField('Until', default="now"),
'row_limit':
SelectField(
'Row limit',
default=config.get("ROW_LIMIT"),
choices=[(s, s) for s in self.row_limits]),
'limit':
SelectField(
'Series limit', choices=[(s, s) for s in self.series_limits],
default=50,
description=(
"Limits the number of time series that get displayed")),
'rolling_type': SelectField(
'Rolling',
choices=[(s, s) for s in ['mean', 'sum', 'std']],
description=(
"Defines a rolling window function to apply")),
'rolling_periods': TextField('Periods', description=(
"Defines the size of the rolling window function, "
"relative to the 'granularity' field")),
'series': SelectField(
'Series', choices=group_by_choices,
description=(
"Defines the grouping of entities. "
"Each serie is shown as a specific color on the chart and "
"has a legend toggle")),
'entity': SelectField('Entity', choices=group_by_choices,
description="This define the element to be plotted on the chart"),
'x': SelectField(
'X Axis', choices=datasource.metrics_combo,
description="Metric assigned to the [X] axis"),
'y': SelectField('Y Axis', choices=datasource.metrics_combo,
description="Metric assigned to the [Y] axis"),
'size': SelectField('Bubble Size', choices=datasource.metrics_combo),
'where': TextField('Custom WHERE clause', default=''),
'compare_lag': TextField('Comparison Period Lag',
description="Based on granularity, number of time periods to compare against"),
'compare_suffix': TextField('Comparison suffix',
description="Suffix to apply after the percentage display"),
'markup_type': SelectField(
"Markup Type",
choices=[(s, s) for s in ['markdown', 'html']],
default="markdown",
description="Pick your favorite markup language"),
'rotation': SelectField(
"Rotation",
choices=[(s, s) for s in ['random', 'flat', 'square']],
default="random",
description="Rotation to apply to words in the cloud"),
'code': TextAreaField("Code", description="Put your code here"),
'size_from': TextField(
"Font Size From",
default="20",
description="Font size for the smallest value in the list"),
'size_to': TextField(
"Font Size To",
default="150",
description="Font size for the biggest value in the list"),
'show_brush': BooleanField(
"Range Selector", default=True,
description="Whether to display the time range interactive selector"),
'show_legend': BooleanField(
"Legend", default=True,
description="Whether to display the legend (toggles)"),
'rich_tooltip': BooleanField(
"Rich Tooltip", default=True,
description="The rich tooltip shows a list of all series for that point in time"),
'y_axis_zero': BooleanField(
"Y Axis Zero", default=False,
description="Force the Y axis to start at 0 instead of the minimum value"),
'y_log_scale': BooleanField(
"Y Log", default=False,
description="Use a log scale for the Y axis"),
'x_log_scale': BooleanField(
"X Log", default=False,
description="Use a log scale for the X axis"),
'donut': BooleanField(
"Donut", default=False,
description="Do you want a donut or a pie?"),
'contribution': BooleanField(
"Contribution", default=False,
description="Compute the contribution to the total"),
'num_period_compare': IntegerField(
"Period Ratio", default=None,
description=(
"Number of period to compare against, "
"this is relative to the granularity selected")),
}
class QueryForm(OmgWtForm):
field_order = copy(viz.form_fields)
css_classes = field_css_classes
def get_form(self, previous=False):
px_form_fields = self.field_dict
viz = self.viz
datasource = viz.datasource
field_css_classes = {k: ['form-control'] for k in px_form_fields.keys()}
select2 = [
'viz_type', 'metrics', 'groupby',
'row_limit', 'rolling_type', 'series',
'entity', 'x', 'y', 'size', 'rotation', 'metric', 'limit',
'markup_type',]
field_css_classes['since'] += ['select2_free_since']
field_css_classes['until'] += ['select2_free_until']
field_css_classes['granularity'] += ['select2_free_granularity']
for field in ('show_brush', 'show_legend', 'rich_tooltip'):
field_css_classes[field] += ['input-sm']
for field in select2:
field_css_classes[field] += ['select2']
for i in range(10):
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
'Filter 1',
choices=[(s, s) for s in datasource.filterable_column_names]))
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
'Filter 1', choices=[(m, m) for m in ['in', 'not in']]))
setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super"))
for ff in viz.form_fields:
if isinstance(ff, basestring):
ff = [ff]
for s in ff:
if s:
setattr(QueryForm, s, px_form_fields[s])
# datasource type specific form elements
if datasource.__class__.__name__ == 'Table':
QueryForm.field_order += ['where']
setattr(QueryForm, 'where', px_form_fields['where'])
return QueryForm
class QueryForm(OmgWtForm):
field_order = copy(viz.form_fields)
css_classes = field_css_classes
standalone = HiddenField()
async = HiddenField()
json = HiddenField()
previous_viz_type = HiddenField()
standalone = HiddenField()
for i in range(10):
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
'Filter 1',
default='',
choices=[(s, s) for s in datasource.filterable_column_names]))
setattr(QueryForm, 'flt_op_' + str(i), SelectField(
'Filter 1',
default='',
choices=[(m, m) for m in ['in', 'not in']]))
setattr(
QueryForm, 'flt_eq_' + str(i),
TextField("Super", default=''))
for ff in viz.form_fields:
if isinstance(ff, basestring):
ff = [ff]
for s in ff:
if s:
setattr(QueryForm, s, px_form_fields[s])
# datasource type specific form elements
if datasource.__class__.__name__ == 'Table':
QueryForm.field_order += ['where']
setattr(QueryForm, 'where', px_form_fields['where'])
return QueryForm

View File

@ -123,6 +123,7 @@
<input type="hidden" name="datasource_name" value="{{ datasource.name }}">
<input type="hidden" name="datasource_id" value="{{ datasource.id }}">
<input type="hidden" name="datasource_type" value="{{ datasource.type }}">
{{ form.previous_viz_type() }}
</form><br>
</div>
@ -132,7 +133,7 @@
<span class="label label-success">
{{ "{0:0.4f}".format(results.duration.total_seconds()) }} s
</span>
<span class="label label-info btn"
<span class="label label-info btn"
data-toggle="modal" data-target="#query_modal">query</span>
{% endif %}
</h3>

View File

@ -3,7 +3,7 @@
{% if viz.form_data.get("json") == "true" %}
{{ viz.get_json() }}
{% else %}
{% if viz.form_data.get("standalone") == "true" %}
{% if viz.request.args.get("standalone") == "true" %}
{% extends 'panoramix/viz_standalone.html' %}
{% else %}
{% extends 'panoramix/datasource.html' %}
@ -16,7 +16,7 @@
{% block head_css %}
{{super()}}
{% if viz.form_data.get("skip_libs") != "true" %}
{% if viz.request.args.get("skip_libs") != "true" %}
{% for css in viz.css_files %}
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename=css) }}">
{% endfor %}
@ -27,11 +27,11 @@
{% block tail %}
{{super()}}
{% if viz.form_data.get("skip_libs") != "true" %}
{% if viz.request.args.get("skip_libs") != "true" %}
{% for js in viz.js_files %}
<script src="{{ url_for('static', filename=js) }}"></script>
{% endfor %}
{{ viz_macros.viz_js(viz) }}
{% endif %}
{{ viz_macros.viz_js(viz) }}
{% endblock %}
{% endif %}

View File

@ -37,7 +37,7 @@
nv.addGraph(function() {
// chart_type is {{ viz.chart_type }}
{% if viz.chart_type == 'line' %}
{% if viz.form_data.show_brush == 'y' %}
{% if viz.form_data.show_brush %}
var chart = nv.models.lineWithFocusChart()
var xext = chart.xAxis.scale().domain();
chart
@ -52,10 +52,10 @@
chart.xAxis
.showMaxMin(false)
.tickFormat(function (d) {return tickMultiFormat(new Date(d)); });
chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }});
chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }});
chart.yAxis.tickFormat(d3.format('.3s'));
{% if viz.form_data.contribution=='y' or viz.form_data.get("num_period_compare") %}
{% if viz.form_data.contribution or viz.form_data.get("num_period_compare") %}
chart.yAxis.tickFormat(d3.format('.3p'));
chart.y2Axis.tickFormat(d3.format('.3p'));
{% endif %}
@ -71,8 +71,8 @@
{% elif viz.chart_type == 'pie' %}
var chart = nv.models.pieChart()
chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }});
{% if viz.form_data.donut=='y' %}
chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }});
{% if viz.form_data.donut %}
chart.donut(true);
chart.donutLabelsOutside(true);
{% endif %}
@ -91,14 +91,14 @@
chart.xAxis
.showMaxMin(false)
.tickFormat(function (d) {return tickMultiFormat(new Date(d)); });
chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }});
chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }});
chart.yAxis.tickFormat(d3.format('.3p'));
{% elif viz.chart_type == 'bubble' %}
var chart = nv.models.scatterChart();
chart.xAxis.tickFormat(d3.format('.3s'));
chart.yAxis.tickFormat(d3.format('.3s'));
chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }});
chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }});
chart.pointRange([5, 5000]);
{% elif viz.chart_type == 'stacked' %}
@ -107,21 +107,21 @@
chart.xAxis
.showMaxMin(false)
.tickFormat(function (d) {return tickMultiFormat(new Date(d)); });
chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }});
chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }});
chart.yAxis.tickFormat(d3.format('.3s'));
{% endif %}
{% if viz.chart_type in ("line", "stacked") and viz.form_data.rich_tooltip == 'y' %}
{% if viz.chart_type in ("line", "stacked") and viz.form_data.rich_tooltip %}
chart.useInteractiveGuideline(true);
{% endif %}
{% if viz.form_data.y_axis_zero == 'y' %}
{% if viz.form_data.y_axis_zero %}
chart.forceY([0, 1]);
{% elif viz.form_data.y_log_scale == 'y' %}
{% elif viz.form_data.y_log_scale %}
chart.yScale(d3.scale.log());
{% endif %}
{% if viz.form_data.x_log_scale == 'y' %}
{% if viz.form_data.x_log_scale %}
chart.xScale(d3.scale.log());
{% endif %}

View File

@ -1,6 +1,6 @@
<html>
<head>
{% if viz.form_data.get("skip_libs") != "true" %}
{% if viz.request.args.get("skip_libs") != "true" %}
{% block head %}
<script src="{{url_for('appbuilder.static',filename='js/jquery-latest.js')}}"></script>
{% endblock %}

View File

@ -1,5 +1,5 @@
{% macro viz_html(viz) %}
{% if viz.form_data.get("async") == "true" %}
{% if viz.request.args.get("async") == "true" %}
{% set df = viz.get_df() %}
<table class="dataframe table table-striped table-bordered table-condensed">
<thead>

View File

@ -3,23 +3,19 @@ from datetime import datetime
import json
import uuid
from flask import flash
from flask import flash, request
from markdown import markdown
from pandas.io.json import dumps
from werkzeug.datastructures import MultiDict
from werkzeug.datastructures import ImmutableMultiDict
from werkzeug.urls import Href
import numpy as np
import pandas as pd
from panoramix import app, utils
from panoramix.forms import form_factory
from panoramix.forms import FormFactory
config = app.config
CHART_ARGS = {
'title': None,
}
class BaseViz(object):
verbose_name = "Base Viz"
@ -32,12 +28,26 @@ class BaseViz(object):
def __init__(self, datasource, form_data):
self.datasource = datasource
form = self.form_class(form_data)
form.validate()
raise
self.form_data = form_data
if isinstance(form_data, MultiDict):
self.form_data = form_data.to_dict(flat=False)
self.request = request
ff = FormFactory(self)
form_class = ff.get_form()
defaults = form_class().data.copy()
if isinstance(form_data, ImmutableMultiDict):
form = form_class(form_data)
else:
form = form_class(**form_data)
data = form.data.copy()
previous_viz_type = form_data.get('previous_viz_type')
if previous_viz_type in viz_types:
data = {
k: form.data[k]
for k in form_data.keys()
if k in viz_types[previous_viz_type].flat_form_fields() and k in form.data}
defaults.update(data)
self.form_data = defaults
self.form_data['previous_viz_type'] = form_data.get("viz_type")
self.token = self.form_data.get(
'token', 'token_' + uuid.uuid4().hex[:8])
@ -48,9 +58,19 @@ class BaseViz(object):
elif k not in as_list and isinstance(v, list) and v:
self.form_data[k] = v[0]
self.metrics = self.form_data.get('metrics') or ['count']
self.metrics = self.form_data.get('metrics') or []
self.groupby = self.form_data.get('groupby') or []
@classmethod
def flat_form_fields(cls):
l = []
for obj in cls.form_fields:
if isinstance(obj, (tuple, list)):
l += [a for a in obj]
else:
l.append(obj)
return l
def get_url(self, **kwargs):
d = self.form_data.copy()
if 'action' in d:
@ -80,7 +100,7 @@ class BaseViz(object):
@property
def form_class(self):
return form_factory(self)
return FormFactory(self).get_form()
def query_filters(self):
form_data = self.form_data
@ -334,11 +354,13 @@ class NVD3TimeSeriesViz(NVD3Viz):
form_data = self.form_data
df = super(NVD3TimeSeriesViz, self).get_df()
df = df.fillna(0)
metrics = self.metrics
if form_data.get("granularity") == "all":
raise Exception("Pick a time granularity for your time series")
df = df.pivot_table(
index="timestamp",
columns=self.groupby,
values=metrics,)
columns=self.form_data.get('groupby'),
values=self.form_data.get('metrics'))
if self.sort_series:
dfs = df.sum()
@ -379,7 +401,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
series_title = name
else:
name = ["{}".format(s) for s in name]
if len(self.metrics) > 1:
if len(self.form_data.get('metrics')) > 1:
series_title = ", ".join(name)
else:
series_title = ", ".join(name[1:])