pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Dynamic time granularity on any datetime column

This commit is contained in:
Maxime Beauchemin 2016-03-14 21:51:35 -07:00
parent 2aa0e0dce0
commit 27fb810dd7
24 changed files with 345 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
*.pyc
babel
.DS_Store
.coverage
_build

3
babel/babel.cfg
View File

@ -1,3 +0,0 @@
[python: **.py]
[jinja2: **/templates/**.html]
encoding = utf-8

1
babel/messages.pot
View File

@ -1 +0,0 @@

2
panoramix/__init__.py
View File

@ -33,4 +33,4 @@ appbuilder = AppBuilder(
sm = appbuilder.sm
get_session = appbuilder.get_session
from panoramix import config, views
from panoramix import config, views # noqa

4
panoramix/assets/javascripts/featured.js
View File

@ -1,9 +1,10 @@
var $ = window.$ = require('jquery');
var jQuery = window.jQuery = $;
var px = require('./modules/panoramix.js');
require('bootstrap');
require('datatables');
require('../node_modules/datatables-bootstrap3-plugin/media/css/datatables-bootstrap3.css');
require('bootstrap');
$(document).ready(function () {
$('#dataset-table').DataTable({
@ -13,5 +14,6 @@ $(document).ready(function () {
]
});
$('#dataset-table_info').remove();
//$('input[type=search]').addClass('form-control'); # TODO get search box to look nice
$('#dataset-table').show();
});

5
panoramix/bin/panoramix
View File

@ -7,6 +7,7 @@ from subprocess import Popen
from flask.ext.script import Manager
from panoramix import app
from flask.ext.migrate import MigrateCommand
import panoramix
from panoramix import db
from panoramix import data, utils
@ -49,7 +50,7 @@ def runserver(debug, port, timeout, workers):
@manager.command
def init():
"""Inits the Panoramix application"""
utils.init()
utils.init(panoramix)
@manager.option(
'-s', '--sample', action='store_true',
@ -58,6 +59,8 @@ def load_examples(sample):
"""Loads a set of Slices and Dashboards and a supporting dataset """
print("Loading examples into {}".format(db))
data.load_css_templates()
print("Loading [World Bank's Health Nutrition and Population Stats]")
data.load_world_bank_health_n_pop()

6
panoramix/config.py
View File

@ -25,7 +25,7 @@ CUSTOM_SECURITY_MANAGER = None
# ---------------------------------------------------------
# Your App secret key
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h' # noqa
# The SQLAlchemy connection string.
SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/panoramix.db'
@ -48,7 +48,7 @@ SHOW_STACKTRACE = True
APP_NAME = "Panoramix"
# Uncomment to setup Setup an App icon
APP_ICON = "/static/img/chaudron_white.png"
# APP_ICON = "/static/img/something.png"
# Druid query timezone
# tz.tzutc() : Using utc timezone
@ -113,6 +113,6 @@ IMG_UPLOAD_URL = '/static/uploads/'
# IMG_SIZE = (300, 200, True)
try:
from panoramix_config import *
from panoramix_config import * # noqa
except Exception:
pass

64
panoramix/data/__init__.py
View File

@ -274,33 +274,14 @@ def load_world_bank_health_n_pop():
dash = Dash(
dashboard_title=dash_name,
position_json=json.dumps(l, indent=4),
slug="world_health",
)
for s in slices:
dash.slices.append(s)
db.session.commit()
def load_birth_names():
session = db.session
with gzip.open(os.path.join(DATA_FOLDER, 'birth_names.json.gz')) as f:
pdf = pd.read_json(f)
pdf.ds = pd.to_datetime(pdf.ds, unit='ms')
pdf.to_sql(
'birth_names',
db.engine,
if_exists='replace',
chunksize=500,
dtype={
'ds': DateTime,
'gender': String(16),
'state': String(10),
'name': String(255),
},
index=False)
l = []
print("Done loading table!")
print("-" * 80)
def load_css_templates():
print('Creating default CSS templates')
CSS = models.CssTemplate
@ -400,6 +381,27 @@ def load_birth_names():
db.session.merge(obj)
db.session.commit()
def load_birth_names():
with gzip.open(os.path.join(DATA_FOLDER, 'birth_names.json.gz')) as f:
pdf = pd.read_json(f)
pdf.ds = pd.to_datetime(pdf.ds, unit='ms')
pdf.to_sql(
'birth_names',
db.engine,
if_exists='replace',
chunksize=500,
dtype={
'ds': DateTime,
'gender': String(16),
'state': String(10),
'name': String(255),
},
index=False)
l = []
print("Done loading table!")
print("-" * 80)
print("Creating table reference")
obj = db.session.query(TBL).filter_by(table_name='birth_names').first()
if not obj:
@ -500,12 +502,15 @@ def load_birth_names():
defaults,
viz_type="markup", markup_type="html",
code="""\
<div style="text-align:center">
<h1>Birth Names Dashboard</h1>
<p>The source dataset came from <a href="https://github.com/hadley/babynames">[here]</a></p>
<img src="http://monblog.system-linux.net/image/tux/baby-tux_overlord59-tux.png">
</div>
"""
<div style="text-align:center">
<h1>Birth Names Dashboard</h1>
<p>
The source dataset came from
<a href="https://github.com/hadley/babynames">[here]</a>
</p>
<img src="http://monblog.system-linux.net/image/tux/baby-tux_overlord59-tux.png">
</div>
"""
)),
Slice(
slice_name="Name Cloud",
@ -531,7 +536,7 @@ def load_birth_names():
merge_slice(slc)
print("Creating a dashboard")
dash = session.query(Dash).filter_by(dashboard_title="Births").first()
dash = db.session.query(Dash).filter_by(dashboard_title="Births").first()
if dash:
db.session.delete(dash)
@ -608,7 +613,8 @@ def load_birth_names():
dash = Dash(
dashboard_title="Births",
position_json=json.dumps(l, indent=4),
slug="births",
)
for s in slices:
dash.slices.append(s)
session.commit()
db.session.commit()

4
panoramix/data/countries.py
View File

@ -1,3 +1,7 @@
"""
This module contains data related to countries and is used for geo mapping
"""
countries = [
{
"name": "Angola",

102
panoramix/forms.py
View File

@ -1,20 +1,21 @@
from wtforms import (
Field, Form, SelectMultipleField, SelectField, TextField, TextAreaField,
Form, SelectMultipleField, SelectField, TextField, TextAreaField,
BooleanField, IntegerField, HiddenField)
from wtforms import validators, widgets
from copy import copy
from panoramix import app
from six import string_types
from collections import OrderedDict
config = app.config
class BetterBooleanField(BooleanField):
"""
Fixes behavior of html forms omitting non checked <input>
(which doesn't distinguish False from NULL/missing )
If value is unchecked, this hidden <input> fills in False value
"""
def __call__(self, **kwargs):
html = super(BetterBooleanField, self).__call__(**kwargs)
html += u'<input type="hidden" name="{}" value="false">'.format(self.name)
@ -22,9 +23,9 @@ class BetterBooleanField(BooleanField):
class SelectMultipleSortableField(SelectMultipleField):
"""
Works along with select2sortable to preserves the sort order
"""
"""Works along with select2sortable to preserves the sort order"""
def iter_choices(self):
d = OrderedDict()
for value, label in self.choices:
@ -39,6 +40,9 @@ class SelectMultipleSortableField(SelectMultipleField):
class FreeFormSelect(widgets.Select):
"""A WTF widget that allows for free form entry"""
def __call__(self, field, **kwargs):
kwargs.setdefault('id', field.id)
if self.multiple:
@ -54,13 +58,20 @@ class FreeFormSelect(widgets.Select):
html.append('</select>')
return widgets.HTMLString(''.join(html))
class FreeFormSelectField(SelectField):
""" A WTF SelectField that allows for free form input """
widget = FreeFormSelect()
def pre_validate(self, form):
return
class OmgWtForm(Form):
"""Panoramixification of the WTForm Form object"""
fieldsets = {}
css_classes = dict()
@ -74,6 +85,7 @@ class OmgWtForm(Form):
class FormFactory(object):
"""Used to create the forms in the explore view dynamically"""
series_limits = [0, 5, 10, 25, 50, 100, 500]
fieltype_class = {
SelectField: 'select2',
@ -231,12 +243,15 @@ class FormFactory(object):
]),
description="Charge in the force layout"),
'granularity_sqla': SelectField(
'Time Column', default=datasource.main_dttm_col,
'Time Column',
default=datasource.main_dttm_col or datasource.any_dttm_col,
choices=self.choicify(datasource.dttm_cols),
description=(
"The time granularity for the visualization. Note that you "
"The time column for the visualization. Note that you "
"can define arbitrary expression that return a DATETIME "
"column in the table editor")),
"column in the table editor. Also note that the "
"filter bellow is applied against this column or "
"expression")),
'resample_rule': FreeFormSelectField(
'Resample Rule', default='',
choices=self.choicify(('1T', '1H', '1D', '7D', '1M', '1AS')),
@ -347,7 +362,9 @@ class FormFactory(object):
"complex expression, parenthesis and anything else "
"supported by the backend it is directed towards.")),
'compare_lag': TextField('Comparison Period Lag',
description="Based on granularity, number of time periods to compare against"),
description=(
"Based on granularity, number of time periods to "
"compare against")),
'compare_suffix': TextField('Comparison suffix',
description="Suffix to apply after the percentage display"),
'x_axis_format': FreeFormSelectField('X axis format',
@ -356,7 +373,8 @@ class FormFactory(object):
('smart_date', 'Adaptative formating'),
("%m/%d/%Y", '"%m/%d/%Y" | 01/14/2019'),
("%Y-%m-%d", '"%Y-%m-%d" | 2019-01-14'),
("%Y-%m-%d %H:%M:%S", '"%Y-%m-%d %H:%M:%S" | 2019-01-14 01:32:10'),
("%Y-%m-%d %H:%M:%S",
'"%Y-%m-%d %H:%M:%S" | 2019-01-14 01:32:10'),
("%H:%M:%S", '"%H:%M:%S" | 01:32:10'),
],
description="D3 format syntax for y axis "
@ -474,12 +492,10 @@ class FormFactory(object):
def choicify(l):
return [("{}".format(obj), "{}".format(obj)) for obj in l]
def get_form(self, previous=False):
px_form_fields = self.field_dict
def get_form(self):
viz = self.viz
datasource = viz.datasource
field_css_classes = {}
for name, obj in px_form_fields.items():
for name, obj in self.field_dict.items():
field_css_classes[name] = ['form-control']
s = self.fieltype_class.get(obj.field_class)
if s:
@ -489,7 +505,7 @@ class FormFactory(object):
field_css_classes[field] += ['input-sm']
class QueryForm(OmgWtForm):
fieldsets = copy(viz.fieldsetizer())
fieldsets = copy(viz.fieldsets)
css_classes = field_css_classes
standalone = HiddenField()
async = HiddenField()
@ -501,7 +517,7 @@ class FormFactory(object):
collapsed_fieldsets = HiddenField()
viz_type = self.field_dict.get('viz_type')
filter_cols = datasource.filterable_column_names or ['']
filter_cols = viz.datasource.filterable_column_names or ['']
for i in range(10):
setattr(QueryForm, 'flt_col_' + str(i), SelectField(
'Filter 1',
@ -514,18 +530,16 @@ class FormFactory(object):
setattr(
QueryForm, 'flt_eq_' + str(i),
TextField("Super", default=''))
for fieldset in viz.fieldsetizer():
for ff in fieldset['fields']:
if ff:
if isinstance(ff, string_types):
ff = [ff]
for s in ff:
if s:
setattr(QueryForm, s, px_form_fields[s])
for field in viz.flat_form_fields():
setattr(QueryForm, field, self.field_dict[field])
def add_to_form(attrs):
for attr in attrs:
setattr(QueryForm, attr, self.field_dict[attr])
# datasource type specific form elements
if datasource.__class__.__name__ == 'SqlaTable':
if viz.datasource.__class__.__name__ == 'SqlaTable':
QueryForm.fieldsets += ({
'label': 'SQL',
'fields': ['where', 'having'],
@ -533,12 +547,36 @@ class FormFactory(object):
"This section exposes ways to include snippets of "
"SQL in your query"),
},)
setattr(QueryForm, 'where', px_form_fields['where'])
setattr(QueryForm, 'having', px_form_fields['having'])
add_to_form(('where', 'having'))
grains = viz.datasource.database.grains()
if 'granularity' in viz.flat_form_fields():
setattr(
QueryForm,
'granularity', px_form_fields['granularity_sqla'])
field_css_classes['granularity'] = ['form-control', 'select2']
if not viz.datasource.any_dttm_col:
return QueryForm
if grains:
time_fields = ('granularity_sqla', 'time_grain_sqla')
self.field_dict['time_grain_sqla'] = SelectField(
'Time Grain',
choices=self.choicify((grain.name for grain in grains)),
default="Time Column",
description=(
"The time granularity for the visualization. This "
"applies a date transformation to alter "
"your time column and defines a new time granularity."
"The options here are defined on a per database "
"engine basis in the Panoramix source code"))
add_to_form(time_fields)
field_css_classes['time_grain_sqla'] = ['form-control', 'select2']
else:
time_fields = 'granularity_sqla'
add_to_form((time_fields, ))
add_to_form(('since', 'until'))
QueryForm.fieldsets = ({
'label': 'Time',
'fields': (
time_fields,
('since', 'until'),
),
'description': "Time related form attributes",
},) + tuple(QueryForm.fieldsets)
field_css_classes['granularity'] = ['form-control', 'select2']
return QueryForm

139
panoramix/migrations/versions/18e88e1cc004_making_audit_nullable.py
View File

@ -15,85 +15,84 @@ import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.alter_column('clusters', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('clusters', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
try:
op.alter_column(
'clusters', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column(
'clusters', 'created_on',
existing_type=sa.DATETIME(), nullable=True)
op.drop_constraint(None, 'columns', type_='foreignkey')
op.drop_constraint(None, 'columns', type_='foreignkey')
op.drop_column('columns', 'created_on')
op.drop_column('columns', 'created_by_fk')
op.drop_column('columns', 'changed_on')
op.drop_column('columns', 'changed_by_fk')
op.alter_column('css_templates', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('css_templates', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dashboards', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dashboards', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.create_unique_constraint(None, 'dashboards', ['slug'])
op.alter_column('datasources', 'changed_by_fk',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('datasources', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('datasources', 'created_by_fk',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('datasources', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dbs', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dbs', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('slices', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('slices', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('sql_metrics', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('sql_metrics', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('table_columns', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('table_columns', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('tables', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('tables', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('url', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('url', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
### end Alembic commands ###
except:
pass
op.alter_column('css_templates', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('css_templates', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dashboards', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dashboards', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.create_unique_constraint(None, 'dashboards', ['slug'])
op.alter_column('datasources', 'changed_by_fk',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('datasources', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('datasources', 'created_by_fk',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('datasources', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dbs', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('dbs', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('slices', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('slices', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('sql_metrics', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('sql_metrics', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('table_columns', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('table_columns', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('tables', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('tables', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('url', 'changed_on',
existing_type=sa.DATETIME(),
nullable=True)
op.alter_column('url', 'created_on',
existing_type=sa.DATETIME(),
nullable=True)
### end Alembic commands ###
def downgrade():

157
panoramix/models.py
View File

@ -1,6 +1,7 @@
from copy import deepcopy, copy
from collections import namedtuple
from datetime import timedelta, datetime
import functools
import json
import logging
from six import string_types
@ -8,7 +9,7 @@ import sqlparse
import requests
from dateutil.parser import parse
from flask import flash
from flask import flash, request, g
from flask.ext.appbuilder import Model
from flask.ext.appbuilder.models.mixins import AuditMixin
import pandas as pd
@ -39,31 +40,39 @@ class AuditMixinNullable(AuditMixin):
changed_on = Column(
DateTime, default=datetime.now,
onupdate=datetime.now, nullable=True)
@declared_attr
def created_by_fk(cls):
return Column(Integer, ForeignKey('ab_user.id'),
default=cls.get_user_id, nullable=True)
@declared_attr
def changed_by_fk(cls):
return Column(Integer, ForeignKey('ab_user.id'),
default=cls.get_user_id, onupdate=cls.get_user_id, nullable=True)
@property
def created_by_(self):
return '{}'.format(self.created_by or '')
@property
@property # noqa
def changed_by_(self):
return '{}'.format(self.changed_by or '')
class Url(Model, AuditMixinNullable):
"""Used for the short url feature"""
__tablename__ = 'url'
id = Column(Integer, primary_key=True)
url = Column(Text)
class CssTemplate(Model, AuditMixinNullable):
"""CSS templates for dashboards"""
__tablename__ = 'css_templates'
id = Column(Integer, primary_key=True)
template_name = Column(String(250))
@ -71,7 +80,9 @@ class CssTemplate(Model, AuditMixinNullable):
class Slice(Model, AuditMixinNullable):
"""A slice is essentially a report or a view on data"""
__tablename__ = 'slices'
id = Column(Integer, primary_key=True)
slice_name = Column(String(250))
@ -154,17 +165,6 @@ class Slice(Model, AuditMixinNullable):
return '<a href="{url}">{self.slice_name}</a>'.format(
url=url, self=self)
@property
def js_files(self):
return viz_types[self.viz_type].js_files
@property
def css_files(self):
return viz_types[self.viz_type].css_files
def get_viz(self):
pass
dashboard_slices = Table('dashboard_slices', Model.metadata,
Column('id', Integer, primary_key=True),
@ -174,7 +174,9 @@ dashboard_slices = Table('dashboard_slices', Model.metadata,
class Dashboard(Model, AuditMixinNullable):
"""A dash to slash"""
__tablename__ = 'dashboards'
id = Column(Integer, primary_key=True)
dashboard_title = Column(String(500))
@ -203,20 +205,6 @@ class Dashboard(Model, AuditMixinNullable):
def dashboard_link(self):
return '<a href="{self.url}">{self.dashboard_title}</a>'.format(self=self)
@property
def js_files(self):
l = []
for o in self.slices:
l += [f for f in o.js_files if f not in l]
return l
@property
def css_files(self):
l = []
for o in self.slices:
l += o.css_files
return list(set(l))
@property
def json_data(self):
d = {
@ -267,6 +255,37 @@ class Database(Model, AuditMixinNullable):
def safe_sqlalchemy_uri(self):
return self.sqlalchemy_uri
def grains(self):
"""Defines time granularity database-specific expressions. The idea
here is to make it easy for users to change the time grain form a
datetime (maybe the source grain is arbitrary timestamps, daily
or 5 minutes increments) to another, "truncated" datetime. Since
each database has slightly different but similar datetime functions,
this allows a mapping between database engines and actual functions.
"""
Grain = namedtuple('Grain', 'name function')
DB_TIME_GRAINS = {
'presto': (
Grain('Time Column', '{col}'),
Grain('week', "date_trunc('week', {col})"),
Grain('month', "date_trunc('month', {col})"),
),
'mysql': (
Grain('Time Column', '{col}'),
Grain('day', 'DATE({col})'),
Grain('week', 'DATE_SUB({col}, INTERVAL DAYOFWEEK({col}) - 1 DAY)'),
Grain('month', 'DATE_SUB({col}, INTERVAL DAYOFMONTH({col}) - 1 DAY)'),
),
}
for db_type, grains in DB_TIME_GRAINS.items():
if self.sqlalchemy_uri.startswith(db_type):
return grains
def grains_dict(self):
return {grain.name: grain for grain in self.grains()}
def get_table(self, table_name):
meta = MetaData()
return Table(
@ -345,6 +364,12 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
l.append(self.main_dttm_col)
return l
@property
def any_dttm_col(self):
cols = self.dttm_cols
if cols:
return cols[0]
@property
def html(self):
t = ((c.column_name, c.type) for c in self.columns)
@ -386,8 +411,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
self, groupby, metrics,
granularity,
from_dttm, to_dttm,
limit_spec=None,
filter=None,
filter=None, # noqa
is_timeseries=True,
timeseries_limit=15, row_limit=None,
inner_from_dttm=None, inner_to_dttm=None,
@ -400,14 +424,21 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
cols = {col.column_name: col for col in self.columns}
qry_start_dttm = datetime.now()
if not self.main_dttm_col:
if not granularity and is_timeseries:
raise Exception(
"Datetime column not provided as part table configuration")
dttm_expr = cols[granularity].expression
if dttm_expr:
"Datetime column not provided as part table configuration "
"and is required by this type of chart")
if granularity:
dttm_expr = cols[granularity].expression or granularity
# Transforming time grain into an expression based on configuration
time_grain_sqla = extras.get('time_grain_sqla')
if time_grain_sqla:
udf = self.database.grains_dict().get(time_grain_sqla, '{col}')
dttm_expr = udf.function.format(col=dttm_expr)
timestamp = literal_column(dttm_expr).label('timestamp')
else:
timestamp = literal_column(granularity).label('timestamp')
metrics_exprs = [
literal_column(m.expression).label(m.metric_name)
for m in self.metrics if m.metric_name in metrics]
@ -455,16 +486,17 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
if not columns:
qry = qry.group_by(*groupby_exprs)
tf = '%Y-%m-%d %H:%M:%S.%f'
time_filter = [
timestamp >= from_dttm.strftime(tf),
timestamp <= to_dttm.strftime(tf),
]
inner_time_filter = copy(time_filter)
if inner_from_dttm:
inner_time_filter[0] = timestamp >= inner_from_dttm.strftime(tf)
if inner_to_dttm:
inner_time_filter[1] = timestamp <= inner_to_dttm.strftime(tf)
if granularity:
tf = '%Y-%m-%d %H:%M:%S.%f'
time_filter = [
timestamp >= from_dttm.strftime(tf),
timestamp <= to_dttm.strftime(tf),
]
inner_time_filter = copy(time_filter)
if inner_from_dttm:
inner_time_filter[0] = timestamp >= inner_from_dttm.strftime(tf)
if inner_to_dttm:
inner_time_filter[1] = timestamp <= inner_to_dttm.strftime(tf)
where_clause_and = []
having_clause_and = []
for col, op, eq in filter:
@ -483,7 +515,8 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
where_clause_and += [text(extras['where'])]
if extras and 'having' in extras:
having_clause_and += [text(extras['having'])]
qry = qry.where(and_(*(time_filter + where_clause_and)))
if granularity:
qry = qry.where(and_(*(time_filter + where_clause_and)))
qry = qry.having(and_(*having_clause_and))
if groupby:
qry = qry.order_by(desc(main_metric_expr))
@ -813,8 +846,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
self, groupby, metrics,
granularity,
from_dttm, to_dttm,
limit_spec=None,
filter=None,
filter=None, # noqa
is_timeseries=True,
timeseries_limit=None,
row_limit=None,
@ -888,7 +920,9 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
pre_qry['limit_spec'] = {
"type": "default",
"limit": timeseries_limit,
'intervals': inner_from_dttm.isoformat() + '/' + inner_to_dttm.isoformat(),
'intervals': (
inner_from_dttm.isoformat() + '/' +
inner_to_dttm.isoformat()),
"columns": [{
"dimension": metrics[0] if metrics else self.metrics[0],
"direction": "descending",
@ -902,7 +936,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
if df is not None and not df.empty:
dims = qry['dimensions']
filters = []
for index, row in df.iterrows():
for _, row in df.iterrows():
fields = []
for dim in dims:
f = Filter.build_filter(Dimension(dim) == row[dim])
@ -970,6 +1004,29 @@ class Log(Model):
user = relationship('User', backref='logs', foreign_keys=[user_id])
dttm = Column(DateTime, default=func.now())
@classmethod
def log_this(cls, f):
"""Decorator to log user actions"""
@functools.wraps(f)
def wrapper(*args, **kwargs):
user_id = None
if g.user:
user_id = g.user.id
d = request.args.to_dict()
d.update(kwargs)
log = cls(
action=f.__name__,
json=json.dumps(d),
dashboard_id=d.get('dashboard_id') or None,
slice_id=d.get('slice_id') or None,
user_id=user_id)
db.session.add(log)
db.session.commit()
return f(*args, **kwargs)
return wrapper
class DruidMetric(Model):
__tablename__ = 'metrics'

BIN
panoramix/static/img/chaudron.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 2.4 KiB

BIN
panoramix/static/img/chaudron_white.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 2.2 KiB

BIN
panoramix/static/img/panoramix.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 165 KiB

BIN
panoramix/static/img/panoramix.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.5 MiB

BIN
panoramix/static/img/tux_panoramix.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 73 KiB

1
panoramix/templates/panoramix/base.html
View File

@ -3,6 +3,7 @@
{% block head_css %}
{{super()}}
<link rel="stylesheet" type="text/css" href="/static/assets/stylesheets/panoramix.css" />
<link rel="icon" type="image/png" href="/static/favicon.png">
{% endblock %}
{% block head_js %}

1
panoramix/templates/panoramix/basic.html
View File

@ -8,6 +8,7 @@
{% block head_meta %}{% endblock %}
{% block head_css %}
<link rel="stylesheet" type="text/css" href="/static/assets/node_modules/font-awesome/css/font-awesome.min.css" />
<link rel="icon" type="image/png" href="/static/favicon.png">
{% endblock %}
{% block head_js %}
<script src="/static/assets/javascripts/dist/css-theme.entry.js"></script>

10
panoramix/templates/panoramix/featured.html
View File

@ -1,4 +1,10 @@
{% extends "panoramix/basic.html" %}
{% block head_js %}
{{ super() }}
<script src="/static/assets/javascripts/dist/featured.entry.js"></script>
{% endblock %}
{% block body %}
<div class="container">
<div class="header">
@ -34,7 +40,3 @@
</div>
{% endblock %}
{% block tail_js %}
{{ super() }}
<script src="/static/assets/javascripts/dist/featured.entry.js"></script>
{% endblock %}

46
panoramix/utils.py
View File

@ -1,16 +1,14 @@
from datetime import datetime
import functools
import hashlib
import functools
import json
import logging
from dateutil.parser import parse
from sqlalchemy.types import TypeDecorator, TEXT
from flask import g, request, Markup
from markdown import markdown as md
import parsedatetime
from panoramix import db
from flask_appbuilder.security.sqla import models as ab_models
class memoized(object):
@ -64,12 +62,14 @@ def parse_human_datetime(s):
True
>>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
True
>>> parse_human_datetime('one year ago').date() == (datetime.now() - relativedelta(years=1) ).date()
>>> year_ago_1 = parse_human_datetime('one year ago').date()
>>> year_ago_2 = (datetime.now() - relativedelta(years=1) ).date()
>>> year_ago_1 == year_ago_2
True
"""
try:
dttm = parse(s)
except:
except Exception:
try:
cal = parsedatetime.Calendar()
dttm = dttm_from_timtuple(cal.parse(s)[0])
@ -154,14 +154,13 @@ class ColorFactory(object):
return self.BNB_COLORS[i % len(self.BNB_COLORS)]
def init():
def init(panoramix):
"""
Inits the Panoramix application with security roles and such
"""
from panoramix import appbuilder
from panoramix import models
from flask_appbuilder.security.sqla import models as ab_models
sm = appbuilder.sm
db = panoramix.db
models = panoramix.models
sm = panoramix.appbuilder.sm
alpha = sm.add_role("Alpha")
admin = sm.add_role("Admin")
@ -178,7 +177,6 @@ def init():
sm.add_permission_role(admin, perm)
gamma = sm.add_role("Gamma")
for perm in perms:
s = perm.permission.name
if(
perm.view_menu.name not in (
'ResetPasswordView',
@ -205,30 +203,6 @@ def init():
merge_perm(sm, 'datasource_access', table_perm)
def log_this(f):
'''
Decorator to log user actions
'''
@functools.wraps(f)
def wrapper(*args, **kwargs):
user_id = None
if g.user:
user_id = g.user.id
from panoramix import models
d = request.args.to_dict()
d.update(kwargs)
log = models.Log(
action=f.__name__,
json=json.dumps(d),
dashboard_id=d.get('dashboard_id') or None,
slice_id=d.get('slice_id') or None,
user_id=user_id)
db.session.add(log)
db.session.commit()
return f(*args, **kwargs)
return wrapper
def datetime_f(dttm):
"""
Formats datetime to take less room is recent

31
panoramix/views.py
View File

@ -20,14 +20,15 @@ from sqlalchemy.sql.expression import TextAsFrom
from panoramix import appbuilder, db, models, viz, utils, app, sm, ascii_art
config = app.config
log_this = models.Log.log_this
def validate_json(form, field):
def validate_json(form, field): # noqa
try:
json.loads(field.data)
except Exception as e:
logging.exception(e)
raise ValidationError("Json isn't valid")
raise ValidationError("json isn't valid")
class DeleteMixin(object):
@ -161,7 +162,9 @@ class TableModelView(PanoramixModelView, DeleteMixin):
base_order = ('changed_on','desc')
description_columns = {
'offset': "Timezone offset (in hours) for this datasource",
'description': Markup("Supports <a href='https://daringfireball.net/projects/markdown/'>markdown</a>"),
'description': Markup(
"Supports <a href='https://daringfireball.net/projects/markdown/'>"
"markdown</a>"),
}
def post_add(self, table):
@ -303,7 +306,9 @@ class DruidDatasourceModelView(PanoramixModelView, DeleteMixin):
base_order = ('datasource_name', 'asc')
description_columns = {
'offset': "Timezone offset (in hours) for this datasource",
'description': Markup("Supports <a href='https://daringfireball.net/projects/markdown/'>markdown</a>"),
'description': Markup(
"Supports <a href='"
"https://daringfireball.net/projects/markdown/'>markdown</a>"),
}
def post_add(self, datasource):
@ -332,7 +337,7 @@ def ping():
class R(BaseView):
@utils.log_this
@log_this
@expose("/<url_id>")
def index(self, url_id):
url = db.session.query(models.Url).filter_by(id=url_id).first()
@ -343,7 +348,7 @@ class R(BaseView):
flash("URL to nowhere...", "danger")
return redirect('/')
@utils.log_this
@log_this
@expose("/shortner/", methods=['POST', 'GET'])
def shortner(self):
url = request.form.get('data')
@ -361,7 +366,7 @@ class Panoramix(BaseView):
@has_access
@expose("/explore/<datasource_type>/<datasource_id>/")
@expose("/datasource/<datasource_type>/<datasource_id>/") # Legacy url
@utils.log_this
@log_this
def explore(self, datasource_type, datasource_id):
if datasource_type == "table":
datasource = (
@ -561,8 +566,8 @@ class Panoramix(BaseView):
dash = qry.first()
# Hack to log the dashboard_id properly, even when getting a slug
@utils.log_this
def dashboard(**kwargs):
@log_this
def dashboard(**kwargs): # noqa
pass
dashboard(dashboard_id=dash.id)
@ -578,7 +583,7 @@ class Panoramix(BaseView):
@has_access
@expose("/sql/<database_id>/")
@utils.log_this
@log_this
def sql(self, database_id):
mydb = db.session.query(
models.Database).filter_by(id=database_id).first()
@ -594,7 +599,7 @@ class Panoramix(BaseView):
@has_access
@expose("/table/<database_id>/<table_name>/")
@utils.log_this
@log_this
def table(self, database_id, table_name):
mydb = db.session.query(
models.Database).filter_by(id=database_id).first()
@ -612,7 +617,7 @@ class Panoramix(BaseView):
@has_access
@expose("/select_star/<database_id>/<table_name>/")
@utils.log_this
@log_this
def select_star(self, database_id, table_name):
mydb = db.session.query(
models.Database).filter_by(id=database_id).first()
@ -627,7 +632,7 @@ class Panoramix(BaseView):
@has_access
@expose("/runsql/", methods=['POST', 'GET'])
@utils.log_this
@log_this
def runsql(self):
session = db.session()
limit = 1000

60
panoramix/viz.py
View File

@ -26,8 +26,6 @@ class BaseViz(object):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'metrics', 'groupby',
)
},)
@ -81,23 +79,16 @@ class BaseViz(object):
s = Markup(s)
return s
def fieldsetizer(self):
"""
Makes form_fields support either a list approach or a fieldsets
approach
"""
return self.fieldsets
@classmethod
def flat_form_fields(cls):
l = set()
for d in cls.fieldsets:
for obj in d['fields']:
if isinstance(obj, (tuple, list)):
l |= {a for a in obj}
if obj and isinstance(obj, (tuple, list)):
l |= {a for a in obj if a}
elif obj:
l.add(obj)
return l
return tuple(l)
def reassignments(self):
pass
@ -111,7 +102,7 @@ class BaseViz(object):
d.update(kwargs)
# Remove unchecked checkboxes because HTML is weird like that
for key in d.keys():
if d[key] == False:
if d[key] is False:
del d[key]
href = Href(
'/panoramix/explore/{self.datasource.type}/'
@ -174,7 +165,8 @@ class BaseViz(object):
form_data = self.form_data
groupby = form_data.get("groupby") or []
metrics = form_data.get("metrics") or ['count']
granularity = form_data.get("granularity")
granularity = \
form_data.get("granularity") or form_data.get("granularity_sqla")
limit = int(form_data.get("limit", 0))
row_limit = int(
form_data.get("row_limit", config.get("ROW_LIMIT")))
@ -193,6 +185,7 @@ class BaseViz(object):
extras = {
'where': form_data.get("where", ''),
'having': form_data.get("having", ''),
'time_grain_sqla': form_data.get("time_grain_sqla", ''),
}
d = {
'granularity': granularity,
@ -259,10 +252,8 @@ class TableViz(BaseViz):
verbose_name = "Table View"
fieldsets = (
{
'label': None,
'label': "Chart Options",
'fields': (
'granularity',
('since', 'until'),
'row_limit',
('include_search', None),
)
@ -322,8 +313,6 @@ class PivotTableViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'groupby',
'columns',
'metrics',
@ -409,8 +398,6 @@ class WordCloudViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'series', 'metric', 'limit',
('size_from', 'size_to'),
'rotation',
@ -447,8 +434,6 @@ class BubbleViz(NVD3Viz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'series', 'entity',
'x', 'y',
'size', 'limit',
@ -515,8 +500,6 @@ class BigNumberViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'metric',
'compare_lag',
'compare_suffix',
@ -567,7 +550,6 @@ class NVD3TimeSeriesViz(NVD3Viz):
{
'label': None,
'fields': (
'granularity', ('since', 'until'),
'metrics',
'groupby', 'limit',
),
@ -743,8 +725,6 @@ class DistributionPieViz(NVD3Viz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'metrics', 'groupby',
'limit',
('donut', 'show_legend'),
@ -777,12 +757,6 @@ class DistributionBarViz(DistributionPieViz):
is_timeseries = False
fieldsets = (
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
)
}, {
'label': 'Chart Options',
'fields': (
'groupby',
@ -803,7 +777,7 @@ class DistributionBarViz(DistributionPieViz):
}
def query_obj(self):
d = super(DistributionPieViz, self).query_obj()
d = super(DistributionPieViz, self).query_obj() # noqa
fd = self.form_data
d['is_timeseries'] = False
gb = fd.get('groupby') or []
@ -818,7 +792,7 @@ class DistributionBarViz(DistributionPieViz):
return d
def get_df(self, query_obj=None):
df = super(DistributionPieViz, self).get_df(query_obj)
df = super(DistributionPieViz, self).get_df(query_obj) # noqa
fd = self.form_data
row = df.groupby(self.groupby).sum()[self.metrics[0]].copy()
@ -863,8 +837,6 @@ class SunburstViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'groupby',
'metric', 'secondary_metric',
'row_limit',
@ -925,8 +897,6 @@ class SankeyViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'groupby',
'metric',
'row_limit',
@ -962,8 +932,6 @@ class DirectedForceViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'groupby',
'metric',
'row_limit',
@ -1004,8 +972,6 @@ class WorldMapViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'entity',
'country_fieldtype',
'metric',
@ -1077,8 +1043,6 @@ class FilterBoxViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'groupby',
'metric',
)
@ -1138,8 +1102,6 @@ class ParallelCoordinatesViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'series',
'metrics',
'secondary_metric',
@ -1170,8 +1132,6 @@ class HeatmapViz(BaseViz):
{
'label': None,
'fields': (
'granularity',
('since', 'until'),
'all_columns_x',
'all_columns_y',
'metric',

3
tests/core_tests.py
View File

@ -5,6 +5,7 @@ import unittest
os.environ['PANORAMIX_CONFIG'] = 'tests.panoramix_test_config'
from flask.ext.testing import LiveServerTestCase, TestCase
import panoramix
from panoramix import app, db, models, utils
BASE_DIR = app.config.get("BASE_DIR")
cli = imp.load_source('cli', BASE_DIR + "/bin/panoramix")
@ -21,7 +22,7 @@ class LiveTest(TestCase):
pass
def test_init(self):
utils.init()
utils.init(panoramix)
def test_load_examples(self):
cli.load_examples(sample=True)