pt
/
superset
mirror of https://github.com/apache/superset.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Stable

This commit is contained in:
Maxime 2015-07-16 00:38:03 +00:00
parent 8431800bd6
commit 29b0bdd00a
8 changed files with 183 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
panoramix/TODO.md
View File

@ -1,3 +1,3 @@
# TODO
* Default URL params per datasource
* Get config metrics to work
* Multi-filters
* multi-metrics

124
panoramix/app/models.py
View File

@ -5,14 +5,9 @@ from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean
from sqlalchemy.orm import relationship
from app import db, utils
from dateutil.parser import parse
"""
You can use the extra Flask-AppBuilder fields and Mixin's
AuditMixin will add automatic timestamp of created and modified by who
import json
"""
client = utils.get_pydruid_client()
class Datasource(Model, AuditMixin):
@ -22,12 +17,11 @@ class Datasource(Model, AuditMixin):
is_featured = Column(Boolean, default=False)
is_hidden = Column(Boolean, default=False)
description = Column(Text)
columns = relationship('Column', backref='datasource')
udfs = relationship('JavascriptUdf', backref='datasource')
default_endpoint = Column(Text)
@property
def metrics(self):
return [col.column_name for col in self.columns if not col.groupby]
def metrics_combo(self):
return [(m.metric_name, m.verbose_name) for m in self.metrics]
def __repr__(self):
return self.datasource_name
@ -37,9 +31,18 @@ class Datasource(Model, AuditMixin):
url = "/panoramix/datasource/{}/".format(self.datasource_name)
return '<a href="{url}">{self.datasource_name}</a>'.format(**locals())
def get_metric_obj(self, metric_name):
return [
m.json_obj for m in self.metrics
if m.metric_name == metric_name
][0]
@classmethod
def latest_metadata(cls, name):
results = client.time_boundary(datasource=name)
print "---" * 100
print name
print results
max_time = results[0]['result']['maxTime']
max_time = parse(max_time)
intervals = (max_time - timedelta(seconds=1)).isoformat() + '/'
@ -47,7 +50,13 @@ class Datasource(Model, AuditMixin):
segment_metadata = client.segment_metadata(
datasource=name,
intervals=intervals)
return segment_metadata[-1]['columns']
print segment_metadata
if segment_metadata:
return segment_metadata[-1]['columns']
def generate_metrics(self):
for col in self.columns:
col.generate_metrics()
@classmethod
def sync_to_db(cls, name):
@ -55,6 +64,8 @@ class Datasource(Model, AuditMixin):
if not datasource:
db.session.add(cls(datasource_name=name))
cols = cls.latest_metadata(name)
if not cols:
return
for col in cols:
col_obj = (
db.session
@ -71,7 +82,7 @@ class Datasource(Model, AuditMixin):
col_obj.filterable = True
if col_obj:
col_obj.type = cols[col]['type']
col_obj.generate_metrics()
db.session.commit()
@property
@ -87,19 +98,21 @@ class Datasource(Model, AuditMixin):
return sorted([c.column_name for c in self.columns if c.filterable])
class JavascriptUdf(Model, AuditMixin):
__tablename__ = 'udfs'
class Metric(Model):
__tablename__ = 'metrics'
id = Column(Integer, primary_key=True)
metric_name = Column(String(512))
verbose_name = Column(String(1024))
metric_type = Column(String(32))
datasource_name = Column(
String(256),
ForeignKey('datasources.datasource_name'))
udf_name = Column(String(256))
column_list = Column(String(1024))
code = Column(Text)
def __repr__(self):
return self.udf_name
datasource = relationship('Datasource', backref='metrics')
json = Column(Text)
@property
def json_obj(self):
return json.loads(self.json)
class Column(Model, AuditMixin):
__tablename__ = 'columns'
@ -107,6 +120,7 @@ class Column(Model, AuditMixin):
datasource_name = Column(
String(256),
ForeignKey('datasources.datasource_name'))
datasource = relationship('Datasource', backref='columns')
column_name = Column(String(256))
is_active = Column(Boolean, default=True)
type = Column(String(32))
@ -120,3 +134,73 @@ class Column(Model, AuditMixin):
def __repr__(self):
return self.column_name
@property
def isnum(self):
return self.type in ('LONG', 'DOUBLE')
def generate_metrics(self):
M = Metric
metrics = []
metrics.append(Metric(
metric_name='count',
verbose_name='COUNT(*)',
metric_type='count',
json=json.dumps({
'type': 'count', 'name': 'count'})
))
if self.datasource.datasource_name == 'platform' and self.column_name=='subject_id':
print((self.column_name, self.type, self.isnum))
if self.sum and self.isnum:
mt = self.type.lower() + 'Sum'
name='sum__' + self.column_name
metrics.append(Metric(
metric_name=name,
metric_type='sum',
verbose_name='SUM({})'.format(self.column_name),
json=json.dumps({
'type': mt, 'name': name, 'fieldName': self.column_name})
))
if self.min and self.isnum:
mt = self.type.lower() + 'Min'
name='min__' + self.column_name
metrics.append(Metric(
metric_name=name,
metric_type='min',
verbose_name='MIN({})'.format(self.column_name),
json=json.dumps({
'type': mt, 'name': name, 'fieldName': self.column_name})
))
if self.max and self.isnum:
mt = self.type.lower() + 'Max'
name='max__' + self.column_name
metrics.append(Metric(
metric_name=name,
metric_type='max',
verbose_name='MAX({})'.format(self.column_name),
json=json.dumps({
'type': mt, 'name': name, 'fieldName': self.column_name})
))
if self.count_distinct:
mt = 'count_distinct'
name='count_distinct__' + self.column_name
metrics.append(Metric(
metric_name=name,
verbose_name='COUNT(DISTINCT {})'.format(self.column_name),
metric_type='count_distinct',
json=json.dumps({
'type': 'cardinality',
'name': name,
'fieldNames': [self.column_name]})
))
for metric in metrics:
m = (
db.session.query(M)
.filter(M.datasource_name==self.datasource_name)
.filter(M.metric_name==metric.metric_name)
.first()
)
metric.datasource_name = self.datasource_name
if not m:
db.session.add(metric)
db.session.commit()

BIN
panoramix/app/static/chaudron_white.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.2 KiB

35
panoramix/app/templates/appbuilder/navbar.html Normal file
View File

@ -0,0 +1,35 @@
{% set menu = appbuilder.menu %}
{% set languages = appbuilder.languages %}
<div class="navbar {{menu.extra_classes}}" role="navigation">
<div class="container">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
{% if appbuilder.app_icon %}
<a class="navbar-brand" style="padding:10px;" href="{{appbuilder.get_url_for_index}}">
<img width="30" src="{{appbuilder.app_icon}}" >
</a>
{% endif %}
<span class="navbar-brand">
<a href="{{appbuilder.get_url_for_index}}">
{{ appbuilder.app_name }}
</a>
</span>
</div>
<div class="navbar-collapse collapse">
<ul class="nav navbar-nav">
{% include 'appbuilder/navbar_menu.html' %}
</ul>
<ul class="nav navbar-nav navbar-right">
{% include 'appbuilder/navbar_right.html' %}
</ul>
</div>
</div>
</div>

3
panoramix/app/templates/panoramix/base.html
View File

@ -4,5 +4,8 @@
{{super()}}
<link rel="shortcut icon" href="{{ url_for('static', filename='chaudron.png') }}">
<style>
.navbar-brand a {
color: white;
}
</style>
{% endblock %}

43
panoramix/app/views.py
View File

@ -38,8 +38,7 @@ def form_factory(datasource, form_args=None):
viz_type = SelectField(
'Viz',
choices=[(k, v.verbose_name) for k, v in viz.viz_types.items()])
metric = SelectField(
'Metric', choices=[(m, m) for m in datasource.metrics])
metric = SelectField('Metric', choices=datasource.metrics_combo)
groupby = SelectMultipleField(
'Group by', choices=[
(s, s) for s in datasource.groupby_column_names])
@ -61,22 +60,33 @@ def form_factory(datasource, form_args=None):
class ColumnInlineView(CompactCRUDMixin, ModelView):
datamodel = SQLAInterface(models.Column)
edit_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
list_columns = ['column_name', 'groupby', 'count_distinct', 'sum', 'min', 'max']
edit_columns = [
'column_name', 'datasource', 'groupby', 'count_distinct',
'sum', 'min', 'max']
list_columns = [
'column_name', 'type', 'groupby', 'count_distinct',
'sum', 'min', 'max']
can_delete = False
appbuilder.add_view_no_menu(ColumnInlineView)
class JavascriptUdfInlineView(CompactCRUDMixin, ModelView):
datamodel = SQLAInterface(models.JavascriptUdf)
edit_columns = ['udf_name', 'column_list', 'code']
appbuilder.add_view_no_menu(JavascriptUdfInlineView)
class MetricInlineView(CompactCRUDMixin, ModelView):
datamodel = SQLAInterface(models.Metric)
list_columns = ['metric_name', 'verbose_name', 'metric_type' ]
edit_columns = [
'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json']
add_columns = [
'metric_name', 'verbose_name', 'metric_type', 'datasource', 'json']
appbuilder.add_view_no_menu(MetricInlineView)
class DatasourceModelView(ModelView):
datamodel = SQLAInterface(models.Datasource)
list_columns = ['datasource_link', 'is_featured' ]
related_views = [ColumnInlineView, JavascriptUdfInlineView]
edit_columns = ['datasource_name', 'description', 'is_featured', 'is_hidden']
list_columns = ['datasource_link', 'is_featured', 'is_hidden']
related_views = [ColumnInlineView, MetricInlineView]
edit_columns = [
'datasource_name', 'description', 'is_featured', 'is_hidden',
'default_endpoint']
page_size = 100
@ -90,13 +100,18 @@ appbuilder.add_view(
class Panoramix(BaseView):
@expose("/datasource/<datasource_name>/")
def datasource(self, datasource_name):
viz_type = request.args.get("viz_type", "table")
viz_type = request.args.get("viz_type")
datasource = (
db.session
.query(models.Datasource)
.filter_by(datasource_name=datasource_name)
.first()
)
if not viz_type and datasource.default_endpoint:
return redirect(datasource.default_endpoint)
if not viz_type:
viz_type = "table"
obj = viz.viz_types[viz_type](
datasource,
form_class=form_factory(datasource, request.args),
@ -107,7 +122,7 @@ class Panoramix(BaseView):
@expose("/refresh_datasources/")
def datasources(self):
def refresh_datasources(self):
import requests
import json
endpoint = (
@ -126,4 +141,6 @@ appbuilder.add_link(
href='/panoramix/refresh_datasources/',
category='Admin',
icon="fa-cogs")
#models.Metric.__table__.drop(db.engine)
db.create_all()

13
panoramix/app/viz.py
View File

@ -23,7 +23,7 @@ class BaseViz(object):
self.datasource = datasource
self.form_class = form_class
self.form_data = form_data
self.metric = form_data.get('metric')
self.metric = form_data.get('metric', 'count')
self.df = self.bake_query()
self.view = view
if self.df is not None:
@ -67,7 +67,10 @@ class BaseViz(object):
args = self.form_data
groupby = args.getlist("groupby") or []
granularity = args.get("granularity")
metric = "count"
aggregations = {
m.metric_name: m.json_obj
for m in ds.metrics if m.metric_name == self.metric
}
limit = int(
args.get("limit", config.ROW_LIMIT)) or config.ROW_LIMIT
since = args.get("since", "all")
@ -77,12 +80,12 @@ class BaseViz(object):
'granularity': granularity or 'all',
'intervals': from_dttm + '/' + datetime.now().isoformat(),
'dimensions': groupby,
'aggregations': {"count": agg.doublesum(metric)},
'aggregations': aggregations,
'limit_spec': {
"type": "default",
"limit": limit,
"columns": [{
"dimension": metric,
"dimension": self.metric,
"direction": "descending",
}],
},
@ -151,9 +154,9 @@ class TimeSeriesViz(HighchartsViz):
columns=[
col for col in df.columns if col not in ["timestamp", metric]],
values=[metric])
chart_js = serialize(
df, kind=self.chart_kind, stacked=self.stacked, **CHART_ARGS)
print self.stacked
return super(TimeSeriesViz, self).render(chart_js=chart_js)
def bake_query(self):

2
panoramix/config.py
View File

@ -34,7 +34,7 @@ CSRF_ENABLED = True
APP_NAME = "Panoramix"
# Uncomment to setup Setup an App icon
#APP_ICON = "static/img/logo.jpg"
APP_ICON = "/static/chaudron_white.png"
#----------------------------------------------------
# AUTHENTICATION CONFIG