From 4cb380d9ba10bf098a0a9a330c91f469d867da95 Mon Sep 17 00:00:00 2001 From: Maxime Date: Tue, 14 Jul 2015 22:15:42 +0000 Subject: [PATCH] Starting over with flask app-builder --- panoramix/app.py | 114 ++++++++++-------- panoramix/templates/panoramix/datasource.html | 2 +- panoramix/viz.py | 16 ++- 3 files changed, 76 insertions(+), 56 deletions(-) diff --git a/panoramix/app.py b/panoramix/app.py index d65f4b76ba..eb3aac7ea5 100644 --- a/panoramix/app.py +++ b/panoramix/app.py @@ -29,51 +29,6 @@ class OmgWtForm(Form): return fields -class DruidDataSource(object): - - def __init__(self, name): - self.name = name - self.cols = self.latest_metadata() - self.col_names = sorted([ - col for col in self.cols.keys() - if not col.startswith("_") and col not in self.metrics]) - - def latest_metadata(self): - results = client.time_boundary(datasource=self.name) - max_time = results[0]['result']['maxTime'] - max_time = parse(max_time) - intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' - intervals += (max_time + timedelta(seconds=1)).isoformat() - segment_metadata = client.segment_metadata( - datasource=self.name, - intervals=intervals) - return segment_metadata[-1]['columns'] - - @property - def metrics(self): - return [ - k for k, v in self.cols.items() - if v['type'] != 'STRING' and not k.startswith('_')] - - def sync_to_db(self): - DS = Datasource - datasource = DS.query.filter_by(datasource_name=self.name).first() - if not datasource: - db.session.add(DS(datasource_name=self.name)) - for col in self.cols: - col_obj = Column.query.filter_by(datasource_name=self.name, column_name=col).first() - datatype = self.cols[col]['type'] - if not col_obj: - col_obj = Column(datasource_name=self.name, column_name=col) - db.session.add(col_obj) - if datatype == "STRING": - col_obj.groupby = True - if col_obj: - col_obj.type = self.cols[col]['type'] - - db.session.commit() - - def form_factory(datasource, form_args=None): grain = ['all', 'none', 'minute', 'hour', 'day'] limits = [0, 5, 10, 25, 50, 100, 500] @@ -95,9 +50,10 @@ def form_factory(datasource, form_args=None): metric = SelectField( 'Metric', choices=[(m, m) for m in datasource.metrics]) groupby = SelectMultipleField( - 'Group by', choices=[(m, m) for m in datasource.col_names]) + 'Group by', choices=[ + (s, s) for s in datasource.groupby_column_names]) granularity = SelectField( - 'Granularity', choices=[(g, g) for g in grain]) + 'Time Granularity', choices=[(g, g) for g in grain]) since = SelectField( 'Since', choices=[(s, s) for s in settings.since_l.keys()], default="all") @@ -105,7 +61,7 @@ def form_factory(datasource, form_args=None): 'Limit', choices=[(s, s) for s in limits]) for i in range(10): setattr(QueryForm, 'flt_col_' + str(i), SelectField( - 'Filter 1', choices=[(m, m) for m in datasource.col_names])) + 'Filter 1', choices=[(s, s) for s in datasource.filterable_column_names])) setattr(QueryForm, 'flt_op_' + str(i), SelectField( 'Filter 1', choices=[(m, m) for m in ['==', '!=', 'in',]])) setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super")) @@ -141,6 +97,55 @@ class Datasource(db.Model): description = db.Column(db.Text) created_dttm = db.Column(db.DateTime, default=db.func.now()) + @property + def metrics(self): + return [col.column_name for col in self.columns if not col.groupby] + + @classmethod + def latest_metadata(cls, name): + results = client.time_boundary(datasource=name) + max_time = results[0]['result']['maxTime'] + max_time = parse(max_time) + intervals = (max_time - timedelta(seconds=1)).isoformat() + '/' + intervals += (max_time + timedelta(seconds=1)).isoformat() + segment_metadata = client.segment_metadata( + datasource=name, + intervals=intervals) + return segment_metadata[-1]['columns'] + + @classmethod + def sync_to_db(cls, name): + datasource = cls.query.filter_by(datasource_name=name).first() + if not datasource: + db.session.add(cls(datasource_name=name)) + cols = cls.latest_metadata(name) + for col in cols: + col_obj = Column.query.filter_by(datasource_name=name, column_name=col).first() + datatype = cols[col]['type'] + if not col_obj: + col_obj = Column(datasource_name=name, column_name=col) + db.session.add(col_obj) + if datatype == "STRING": + col_obj.groupby = True + col_obj.filterable = True + if col_obj: + col_obj.type = cols[col]['type'] + + db.session.commit() + + @property + def column_names(self): + return sorted([c.column_name for c in self.columns]) + + @property + def groupby_column_names(self): + return sorted([c.column_name for c in self.columns if c.groupby]) + + @property + def filterable_column_names(self): + return sorted([c.column_name for c in self.columns if c.filterable]) + + class Column(db.Model): __tablename__ = 'columns' @@ -156,6 +161,7 @@ class Column(db.Model): sum = db.Column(db.Boolean, default=False) max = db.Column(db.Boolean, default=False) min = db.Column(db.Boolean, default=False) + filterable = db.Column(db.Boolean, default=False) datasource = db.relationship('Datasource', backref=db.backref('columns', lazy='dynamic')) @@ -193,7 +199,12 @@ class DatasourceView(BaseView): @expose("/datasource//") def datasource(self, datasource_name): viz_type = request.args.get("viz_type", "table") - datasource = DruidDataSource(datasource_name) + datasource = ( + Datasource + .query + .filter_by(datasource_name=datasource_name) + .first() + ) obj = viz.viz_types[viz_type]( datasource, form_class=form_factory(datasource, request.args), @@ -205,7 +216,7 @@ class DatasourceView(BaseView): @expose("/datasources/") - def datasources(): + def datasources(self): import requests import json endpoint = ( @@ -214,8 +225,7 @@ class DatasourceView(BaseView): ).format(**settings.__dict__) datasources = json.loads(requests.get(endpoint).text) for datasource in datasources: - ds = DruidDataSource(datasource) - ds.sync_to_db() + Datasource.sync_to_db(datasource) return json.dumps(datasources, indent=4) diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/templates/panoramix/datasource.html index ea8ce0b467..8f6a45fe92 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/templates/panoramix/datasource.html @@ -16,7 +16,7 @@ form .col {

- {{ datasource.name }} + {{ datasource.datasource_name }}

diff --git a/panoramix/viz.py b/panoramix/viz.py index 0c9285a290..7e4a9c1c9c 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -72,7 +72,7 @@ class BaseViz(object): since = args.get("since", "all") from_dttm = (datetime.now() - settings.since_l[since]).isoformat() d = { - 'datasource': ds.name, + 'datasource': ds.datasource_name, 'granularity': granularity or 'all', 'intervals': from_dttm + '/' + datetime.now().isoformat(), 'dimensions': groupby, @@ -135,6 +135,7 @@ class HighchartsViz(BaseViz): verbose_name = "Base Highcharts Viz" template = 'panoramix/viz_highcharts.html' chart_kind = 'line' + stacked = False class TimeSeriesViz(HighchartsViz): @@ -149,7 +150,9 @@ class TimeSeriesViz(HighchartsViz): columns=[ col for col in df.columns if col not in ["timestamp", metric]], values=[metric]) - chart_js = serialize(df, kind=self.chart_kind, **CHART_ARGS) + chart_js = serialize( + df, kind=self.chart_kind, stacked=self.stacked, **CHART_ARGS) + print self.stacked return super(TimeSeriesViz, self).render(chart_js=chart_js) def bake_query(self): @@ -183,7 +186,8 @@ class TimeSeriesViz(HighchartsViz): class TimeSeriesAreaViz(TimeSeriesViz): - verbose_name = "Time Series - Area Chart" + verbose_name = "Time Series - Stacked Area Chart" + stacked=True chart_kind = "area" @@ -191,6 +195,11 @@ class TimeSeriesBarViz(TimeSeriesViz): verbose_name = "Time Series - Bar Chart" chart_kind = "bar" +class TimeSeriesStackedBarViz(TimeSeriesViz): + verbose_name = "Time Series - Stacked Bar Chart" + chart_kind = "bar" + stacked = True + class DistributionBarViz(HighchartsViz): verbose_name = "Distribution - Bar Chart" @@ -242,4 +251,5 @@ viz_types = OrderedDict([ ['bar', TimeSeriesBarViz], ['dist_bar', DistributionBarViz], ['pie', DistributionPieViz], + ['stacked_ts_bar', TimeSeriesStackedBarViz], ])